From 4a4a7a23ab9a3461f5cd2ad21960ebfd9c8b8825 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 23 Oct 2024 02:36:04 +0200 Subject: [PATCH] Initial commit --- .github/workflows/go.yml | 28 +++++++ .gitignore | 12 +++ LICENSE | 21 +++++ README.md | 6 ++ go.mod | 22 ++++++ go.sum | 68 ++++++++++++++++ pkg/ftp/ftp.go | 117 ++++++++++++++++++++++++++++ pkg/local/local.go | 108 ++++++++++++++++++++++++++ pkg/local/local_test.go | 60 +++++++++++++++ pkg/s3/s3.go | 162 +++++++++++++++++++++++++++++++++++++++ pkg/ssh/ssh.go | 119 ++++++++++++++++++++++++++++ pkg/storage.go | 14 ++++ 12 files changed, 737 insertions(+) create mode 100644 .github/workflows/go.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/ftp/ftp.go create mode 100644 pkg/local/local.go create mode 100644 pkg/local/local_test.go create mode 100644 pkg/s3/s3.go create mode 100644 pkg/ssh/ssh.go create mode 100644 pkg/storage.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..4c7c144 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.21.0' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3ce5949 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +/.history +backup +data +compose.yaml +.env +test.md +.DS_Store +pg-bkup +/.idea +bin +Makefile +tests \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4d26641 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a29244b --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Go Storage + +- Local +- S3 +- SSH +- FTP diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f1a1873 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module github.com/jkaninda/go-storage + +go 1.21.0 + +require ( + github.com/spf13/pflag v1.0.5 +) + +require ( + github.com/aws/aws-sdk-go v1.55.3 // indirect + github.com/bramvdbogaerde/go-scp v1.5.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hpcloud/tail v1.0.0 // indirect + github.com/jlaffaye/ftp v0.2.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/sys v0.22.0 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..359196b --- /dev/null +++ b/go.sum @@ -0,0 +1,68 @@ +github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= +github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= +github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= +github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= +github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/ftp/ftp.go b/pkg/ftp/ftp.go new file mode 100644 index 0000000..4129e58 --- /dev/null +++ b/pkg/ftp/ftp.go @@ -0,0 +1,117 @@ +package ftp + +import ( + "fmt" + "github.com/jkaninda/go-storage/pkg" + "github.com/jlaffaye/ftp" + "io" + "os" + "path/filepath" + "time" +) + +type ftpStorage struct { + *pkg.Backend + client *ftp.ServerConn +} + +// Config holds the SSH connection details +type Config struct { + Host string + User string + Password string + Port string + LocalPath string + RemotePath string +} + +// createClient creates FTP Client +func createClient(conf Config) (*ftp.ServerConn, error) { + ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return nil, fmt.Errorf("failed to connect to FTP: %w", err) + } + + err = ftpClient.Login(conf.User, conf.Password) + if err != nil { + return nil, fmt.Errorf("failed to log in to FTP: %w", err) + } + + return ftpClient, nil +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + client, err := createClient(conf) + if err != nil { + return nil, err + } + return &ftpStorage{ + client: client, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to the remote server +func (s ftpStorage) Copy(fileName string) error { + ftpClient := s.client + defer ftpClient.Quit() + + filePath := filepath.Join(s.LocalPath, fileName) + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", fileName, err) + } + defer file.Close() + + remoteFilePath := filepath.Join(s.RemotePath, fileName) + err = ftpClient.Stor(remoteFilePath, file) + if err != nil { + return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err) + } + + return nil +} + +// CopyFrom copies a file from the remote server to local storage +func (s ftpStorage) CopyFrom(fileName string) error { + ftpClient := s.client + + defer ftpClient.Quit() + + remoteFilePath := filepath.Join(s.RemotePath, fileName) + r, err := ftpClient.Retr(remoteFilePath) + if err != nil { + return fmt.Errorf("failed to retrieve file %s: %w", fileName, err) + } + defer r.Close() + + localFilePath := filepath.Join(s.LocalPath, fileName) + outFile, err := os.Create(localFilePath) + if err != nil { + return fmt.Errorf("failed to create local file %s: %w", fileName, err) + } + defer outFile.Close() + + _, err = io.Copy(outFile, r) + if err != nil { + return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err) + } + + return nil +} + +// Prune deletes old backup created more than specified days +func (s ftpStorage) Prune(retentionDays int) error { + fmt.Println("Deleting old backup from a remote server is not implemented yet") + return nil + +} + +// Name returns the storage name +func (s ftpStorage) Name() string { + return "ftp" +} diff --git a/pkg/local/local.go b/pkg/local/local.go new file mode 100644 index 0000000..448fb90 --- /dev/null +++ b/pkg/local/local.go @@ -0,0 +1,108 @@ +package local + +import ( + "github.com/jkaninda/go-storage/pkg" + "io" + "os" + "path/filepath" + "time" +) + +type localStorage struct { + *pkg.Backend +} +type Config struct { + LocalPath string + RemotePath string +} + +// NewStorage creates new Storage +func NewStorage(conf Config) pkg.Storage { + return &localStorage{ + Backend: &pkg.Backend{ + LocalPath: conf.LocalPath, + RemotePath: conf.RemotePath, + }, + } +} + +// Copy copies file to the local destination path +func (l localStorage) Copy(file string) error { + if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) { + return err + } + err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file)) + if err != nil { + return err + } + return nil +} + +// CopyFrom copies file from a Path to local path +func (l localStorage) CopyFrom(file string) error { + if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) { + return err + } + err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file)) + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (l localStorage) Prune(retentionDays int) error { + currentTime := time.Now() + // Delete file + deleteFile := func(filePath string) error { + err := os.Remove(filePath) + return err + } + // Walk through the directory and delete files modified more than specified days ago + err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + // Check if it's a regular file and if it was modified more than specified days ago + if fileInfo.Mode().IsRegular() { + timeDiff := currentTime.Sub(fileInfo.ModTime()) + if timeDiff.Hours() > 24*float64(retentionDays) { + err := deleteFile(filePath) + if err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + return nil +} + +// Name returns the storage name +func (l localStorage) Name() string { + return "local" +} + +// copyFile copies file +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + out.Close() + return err + } + return out.Close() +} diff --git a/pkg/local/local_test.go b/pkg/local/local_test.go new file mode 100644 index 0000000..73336ee --- /dev/null +++ b/pkg/local/local_test.go @@ -0,0 +1,60 @@ +package local + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +const content = "Lorem ipsum dolor sit amet. Eum eius voluptas sit vitae vitae aut sequi molestias hic accusamus consequatur" +const inputFile = "file.txt" +const localPath = "./tests/local" +const RemotePath = "./tests/remote" + +func TestCopy(t *testing.T) { + + err := os.MkdirAll(localPath, 0777) + if err != nil { + t.Error(err) + } + err = os.MkdirAll(RemotePath, 0777) + if err != nil { + t.Error(err) + } + + _, err = createFile(filepath.Join(localPath, inputFile), content) + if err != nil { + t.Error(err) + } + + l := NewStorage(Config{ + LocalPath: "./tests/local", + RemotePath: "./tests/remote", + }) + err = l.Copy(inputFile) + if err != nil { + t.Error(err) + } + fmt.Printf("File copied to %s\n", filepath.Join(RemotePath, inputFile)) +} +func createFile(fileName, content string) ([]byte, error) { + // Create a file named hello.txt + file, err := os.Create(fileName) + if err != nil { + fmt.Println("Error creating file:", err) + return nil, err + } + defer file.Close() + + // Write the message to the file + _, err = file.WriteString(content) + if err != nil { + fmt.Println("Error writing to file:", err) + return nil, err + } + + fmt.Printf("Successfully wrote to %s\n", fileName) + fileBytes, err := os.ReadFile(fileName) + return fileBytes, err +} diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go new file mode 100644 index 0000000..262ff33 --- /dev/null +++ b/pkg/s3/s3.go @@ -0,0 +1,162 @@ +package s3 + +import ( + "bytes" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/jkaninda/go-storage/pkg" + "net/http" + "os" + "path/filepath" + "time" +) + +type s3Storage struct { + *pkg.Backend + client *session.Session + bucket string +} + +// Config holds the AWS S3 config +type Config struct { + Endpoint string + Bucket string + AccessKey string + SecretKey string + Region string + DisableSsl bool + ForcePathStyle bool + LocalPath string + RemotePath string +} + +// CreateSession creates a new AWS session +func createSession(conf Config) (*session.Session, error) { + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""), + Endpoint: aws.String(conf.Endpoint), + Region: aws.String(conf.Region), + DisableSSL: aws.Bool(conf.DisableSsl), + S3ForcePathStyle: aws.Bool(conf.ForcePathStyle), + } + + return session.NewSession(s3Config) +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + sess, err := createSession(conf) + if err != nil { + return nil, err + } + return &s3Storage{ + client: sess, + bucket: conf.Bucket, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to S3 storage +func (s s3Storage) Copy(fileName string) error { + svc := s3.New(s.client) + file, err := os.Open(filepath.Join(s.LocalPath, fileName)) + if err != nil { + return err + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return err + } + objectKey := filepath.Join(s.RemotePath, fileName) + buffer := make([]byte, fileInfo.Size()) + file.Read(buffer) + fileBytes := bytes.NewReader(buffer) + fileType := http.DetectContentType(buffer) + + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + Body: fileBytes, + ContentLength: aws.Int64(fileInfo.Size()), + ContentType: aws.String(fileType), + }) + if err != nil { + return err + } + + return nil +} + +// CopyFrom copies a file from S3 to local storage +func (s s3Storage) CopyFrom(fileName string) error { + file, err := os.Create(filepath.Join(s.LocalPath, fileName)) + if err != nil { + return err + } + defer file.Close() + + objectKey := filepath.Join(s.RemotePath, fileName) + + downloader := s3manager.NewDownloader(s.client) + _, err = downloader.Download(file, + &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + }) + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (s s3Storage) Prune(retentionDays int) error { + svc := s3.New(s.client) + + // Get the current time + now := time.Now() + backupRetentionDays := now.AddDate(0, 0, -retentionDays) + + // List objects in the bucket + listObjectsInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(s.RemotePath), + } + err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, object := range page.Contents { + if object.LastModified.Before(backupRetentionDays) { + // Object is older than retention days, delete it + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: object.Key, + }) + if err != nil { + fmt.Printf("failed to delete object %s: %v", *object.Key, err) + } else { + fmt.Printf("Deleted object %s", *object.Key) + } + } + } + return !lastPage + }) + if err != nil { + return fmt.Errorf("failed to list objects: %v", err) + } + + return nil + +} + +// Name returns the storage name +func (s s3Storage) Name() string { + return "s3" +} diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go new file mode 100644 index 0000000..d5ac96b --- /dev/null +++ b/pkg/ssh/ssh.go @@ -0,0 +1,119 @@ +package ssh + +import ( + "context" + "errors" + "fmt" + "github.com/bramvdbogaerde/go-scp" + "github.com/bramvdbogaerde/go-scp/auth" + "github.com/jkaninda/go-storage/pkg" + "golang.org/x/crypto/ssh" + "os" + "path/filepath" +) + +type sshStorage struct { + *pkg.Backend + client scp.Client +} + +// Config holds the SSH connection details +type Config struct { + Host string + User string + Password string + Port string + IdentifyFile string + LocalPath string + RemotePath string +} + +// createClient creates SSH Client +func createClient(conf Config) (scp.Client, error) { + if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) { + clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey()) + return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err + } else { + if conf.Password == "" { + return scp.Client{}, errors.New("ssh password required") + } + clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey()) + return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err + + } +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + client, err := createClient(conf) + if err != nil { + return nil, err + } + return &sshStorage{ + client: client, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to the remote server +func (s sshStorage) Copy(fileName string) error { + client := s.client + // Connect to the remote server + err := client.Connect() + if err != nil { + return errors.New("couldn't establish a connection to the remote server") + } + // Open the local file + filePath := filepath.Join(s.LocalPath, fileName) + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer client.Close() + // Copy file to the remote server + err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655") + if err != nil { + return fmt.Errorf("failed to copy file to remote server: %w", err) + } + + return nil +} + +// CopyFrom copies a file from the remote server to local storage +func (s sshStorage) CopyFrom(fileName string) error { + // Create a new SCP client + client := s.client + // Connect to the remote server + err := client.Connect() + if err != nil { + return errors.New("couldn't establish a connection to the remote server") + } + // Close client connection after the file has been copied + defer client.Close() + file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777) + if err != nil { + return errors.New("couldn't open the output file") + } + defer file.Close() + + err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName)) + + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (s sshStorage) Prune(retentionDays int) error { + fmt.Println("Deleting old backup from a remote server is not implemented yet") + return nil +} + +// Name returns the storage name +func (s sshStorage) Name() string { + return "ssh" +} diff --git a/pkg/storage.go b/pkg/storage.go new file mode 100644 index 0000000..16169e0 --- /dev/null +++ b/pkg/storage.go @@ -0,0 +1,14 @@ +package pkg + +type Storage interface { + Copy(fileName string) error + CopyFrom(fileName string) error + Prune(retentionDays int) error + Name() string +} +type Backend struct { + //Local Path + LocalPath string + //Remote path or Destination path + RemotePath string +}