27 Commits

Author SHA1 Message Date
fdbf0cd233 feat: add gsc and go_storage package 2026-02-21 08:11:49 +01:00
Jonas Kaninda
e5e68bdadf Merge pull request #17 from jkaninda/dependabot/go_modules/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob-1.6.1
Some checks failed
Go / build (push) Failing after 19m56s
Lint / Run on Ubuntu (push) Successful in 18m48s
build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob from 1.5.0 to 1.6.1
2025-05-17 13:52:51 +02:00
dependabot[bot]
652d00ce3b build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob
All checks were successful
Lint / Run on Ubuntu (push) Successful in 18m55s
Bumps [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) from 1.5.0 to 1.6.1.
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.5.0...sdk/azcore/v1.6.1)

---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob
  dependency-version: 1.6.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-05-16 19:24:30 +00:00
Jonas Kaninda
c39d828fe6 Merge pull request #18 from jkaninda/dependabot/go_modules/golang.org/x/crypto-0.38.0
All checks were successful
Go / build (push) Successful in 9m43s
Lint / Run on Ubuntu (push) Successful in 18m51s
build(deps): bump golang.org/x/crypto from 0.31.0 to 0.38.0
2025-05-16 21:23:07 +02:00
dependabot[bot]
b9d06c6529 build(deps): bump golang.org/x/crypto from 0.31.0 to 0.38.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.31.0 to 0.38.0.
- [Commits](https://github.com/golang/crypto/compare/v0.31.0...v0.38.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.38.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-05-12 00:38:17 +00:00
Jonas Kaninda
373e8ada7a Merge pull request #16 from jkaninda/dependabot/go_modules/github.com/aws/aws-sdk-go-1.55.7
All checks were successful
Go / build (push) Successful in 9m39s
Lint / Run on Ubuntu (push) Successful in 18m35s
build(deps): bump github.com/aws/aws-sdk-go from 1.55.6 to 1.55.7
2025-04-28 21:17:10 +02:00
dependabot[bot]
da0fc37905 build(deps): bump github.com/aws/aws-sdk-go from 1.55.6 to 1.55.7
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.55.6 to 1.55.7.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG_PENDING.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.6...v1.55.7)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-version: 1.55.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-04-28 00:34:35 +00:00
Jonas Kaninda
cd01e0007c Merge pull request #9 from jkaninda/dependabot/go_modules/github.com/aws/aws-sdk-go-1.55.6
All checks were successful
Go / build (push) Successful in 9m39s
Lint / Run on Ubuntu (push) Successful in 18m36s
build(deps): bump github.com/aws/aws-sdk-go from 1.55.5 to 1.55.6
2025-03-15 15:41:30 +01:00
dependabot[bot]
bcdcffed55 build(deps): bump github.com/aws/aws-sdk-go from 1.55.5 to 1.55.6
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.55.5 to 1.55.6.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG_PENDING.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.5...v1.55.6)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-20 00:36:00 +00:00
Jonas Kaninda
624c986a23 Merge pull request #5 from jkaninda/dependabot/go_modules/github.com/aws/aws-sdk-go-1.55.5
All checks were successful
Go / build (push) Successful in 9m50s
Lint / Run on Ubuntu (push) Successful in 18m33s
build(deps): bump github.com/aws/aws-sdk-go from 1.55.3 to 1.55.5
2024-12-28 07:12:44 +01:00
Jonas Kaninda
a06c6acdb8 Merge pull request #7 from jkaninda/dependabot/go_modules/golang.org/x/crypto-0.31.0
build(deps): bump golang.org/x/crypto from 0.28.0 to 0.31.0
2024-12-28 07:12:33 +01:00
dependabot[bot]
538df008f7 build(deps): bump golang.org/x/crypto from 0.28.0 to 0.31.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.28.0 to 0.31.0.
- [Commits](https://github.com/golang/crypto/compare/v0.28.0...v0.31.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-16 00:29:49 +00:00
dependabot[bot]
10e9be1f49 build(deps): bump github.com/aws/aws-sdk-go from 1.55.3 to 1.55.5
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.55.3 to 1.55.5.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.3...v1.55.5)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-09 00:36:50 +00:00
Jonas Kaninda
367de149f2 Merge pull request #4 from jkaninda/develop
Update readme
2024-12-07 19:47:26 +01:00
1e36fb1884 Update readme 2024-12-07 19:47:07 +01:00
Jonas Kaninda
da738264ab Merge pull request #3 from jkaninda/develop
Fix ssh Identity file
2024-12-07 19:45:54 +01:00
085bdf468f Fix ssh identityfile 2024-12-07 19:44:57 +01:00
Jonas Kaninda
0a70da956f Merge pull request #2 from jkaninda/develop
refactor: refactoring of code to meet all go lint requirements
2024-12-06 16:26:42 +01:00
Jonas Kaninda
793b04340e refactor: refactoring of code to meet all go lint requirements 2024-12-06 16:25:16 +01:00
Jonas Kaninda
1b147ebea5 Merge pull request #1 from jkaninda/develop
feat: add Azure Blob storage
2024-12-06 16:14:21 +01:00
Jonas Kaninda
2bcfd3aacf feat: add Azure Blob storage 2024-12-06 16:13:24 +01:00
Jonas Kaninda
845e0cc43b feat: add Azure Blob storage 2024-12-06 16:10:29 +01:00
Jonas Kaninda
d314c3854b Set up automated updates for Go packages 2024-10-23 10:08:19 +02:00
Jonas Kaninda
2339db673d Create dependabot.yml 2024-10-23 09:47:34 +02:00
Jonas Kaninda
3602c78e73 docs: update readme 2024-10-23 03:52:46 +02:00
Jonas Kaninda
16be2fa70d docs: update readme 2024-10-23 03:52:05 +02:00
Jonas Kaninda
7a2450207e docs: update readme 2024-10-23 02:46:31 +02:00
22 changed files with 2392 additions and 82 deletions

6
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: gomod
directory: /
schedule:
interval: weekly

23
.github/workflows/lint.yml vendored Normal file
View File

@@ -0,0 +1,23 @@
name: Lint
on:
push:
pull_request:
jobs:
lint:
name: Run on Ubuntu
runs-on: ubuntu-latest
steps:
- name: Clone the code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '~1.24'
- name: Run linter
uses: golangci/golangci-lint-action@v7
with:
version: v2.1.2

47
.golangci.yml Normal file
View File

@@ -0,0 +1,47 @@
version: "2"
run:
allow-parallel-runners: true
linters:
default: none
enable:
- copyloopvar
- dupl
- errcheck
- ginkgolinter
- goconst
- gocyclo
- govet
- ineffassign
#- lll
- misspell
- nakedret
- prealloc
- revive
- staticcheck
- unconvert
- unparam
- unused
settings:
revive:
rules:
- name: comment-spacings
exclusions:
generated: lax
rules:
- linters:
- dupl
- lll
path: /*
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$

120
README.md
View File

@@ -1,6 +1,126 @@
# Go Storage
A simple Go storage client
Supported storage:
- Local
- S3
- SSH
- FTP
- Azure Blob
```go
go get github.com/jkaninda/go-storage
```
### Local Storage
```go
localStorage := local.NewStorage(local.Config{
LocalPath: tmpPath,
RemotePath: backupDestination,
})
err = localStorage.Copy(finalFileName)
if err != nil {
log.Fatal("Error copying file, error %v", err)
}
```
### S3 Storage
```go
s3Storage, err := s3.NewStorage(s3.Config{
Endpoint: "",
Bucket: "",
AccessKey: "",
SecretKey: "",
Region: "",
DisableSsl: "",
ForcePathStyle: "",
RemotePath: "",
LocalPath: "",
})
if err != nil {
log.Fatalf("Error creating s3 storage: %s", err)
}
// Copy file to S3
err = s3Storage.Copy(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
// Download file from S3
err = s3Storage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
### SSH Storage
```go
sshStorage, err := ssh.NewStorage(ssh.Config{
Host: "",
Port: 22,
User: "",
Password: "",
RemotePath: "",
LocalPath: "",
})
if err != nil {
log.Fatalf("Error creating SSH storage: %s", err)
}
// Copy file to the remote server
err = sshStorage.Copy(finalFileName)
if err != nil {
log.Fatalf("Error copying backup file: %s", err)
}
// Download file from SSH remote server
err = sshStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
### FTP Storage
```go
ftpStorage, err := ftp.NewStorage(ftp.Config{
Host: "",
Port: 21,
User: "",
Password: "",
RemotePath: "",
LocalPath: "",
})
if err != nil {
log.Fatalf("Error creating FTP storage: %s", err)
}
err = ftpStorage.Copy(finalFileName)
if err != nil {
log.Fatalf("Error copying backup file: %s", err)
}
// Download file from ftp remote server
err = ftpStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```
### Azure Blob storage
```go
azureStorage, err := azure.NewStorage(azure.Config{
ContainerName: '',
AccountName: '',
AccountKey: '',
RemotePath: '',
LocalPath: '',
})
if err != nil {
log.Fatal("Error creating Azure Blob storage storage: %s", err)
}
err = azureStorage.Copy(finalFileName)
if err != nil {
log.Fatal("Error copying file: %s", err)
}
// Download file from Azure Blob remote server
err = azureStorage.CopyFrom(finalFileName)
if err != nil {
log.Fatalf("Error copying file, error %v", err)
}
```

155
azure.go Normal file
View File

@@ -0,0 +1,155 @@
package go_storage
import (
"context"
"fmt"
"os"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
)
// AzureStorage implements Storage interface for Azure Blob Storage.
type AzureStorage struct {
client *azblob.Client
container string
}
// NewAzureStorage creates a new Azure Blob Storage instance.
func NewAzureStorage(config Config) (*AzureStorage, error) {
if err := validateConfig(config, map[string]string{
"container": config.AzureContainer,
}); err != nil {
return nil, fmt.Errorf("azure: %w", err)
}
// Try to get credentials from environment if not provided
account := config.AzureAccount
if account == "" {
account = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
key := config.AzureKey
if key == "" {
key = os.Getenv("AZURE_STORAGE_KEY")
}
endpoint := config.AzureEndpoint
if endpoint == "" {
endpoint = "blob.core.windows.net"
}
var client *azblob.Client
var err error
if account == "" {
// Anonymous access
client, err = azblob.NewClientWithNoCredential(endpoint, nil)
if err != nil {
return nil, fmt.Errorf("azure: failed to create anonymous client: %w", err)
}
} else {
// Authenticated access
credential, err := azblob.NewSharedKeyCredential(account, key)
if err != nil {
return nil, fmt.Errorf("azure: failed to create credentials: %w", err)
}
url := fmt.Sprintf("https://%s.%s", account, endpoint)
client, err = azblob.NewClientWithSharedKeyCredential(url, credential, nil)
if err != nil {
return nil, fmt.Errorf("azure: failed to create client: %w", err)
}
}
return &AzureStorage{
client: client,
container: config.AzureContainer,
}, nil
}
// Upload uploads a file to Azure Blob Storage.
func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("azure: failed to open local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("azure: failed to close file: %v\n", err)
}
}(file)
_, err = a.client.UploadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
if err != nil {
return fmt.Errorf("azure: failed to upload to container %s: %w", a.container, err)
}
return nil
}
// Download downloads a file from Azure Blob Storage.
func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("azure: failed to create local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("azure: failed to close file: %v\n", err)
}
}(file)
_, err = a.client.DownloadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
if err != nil {
return fmt.Errorf("azure: failed to download from container %s: %w", a.container, err)
}
return nil
}
// List lists blobs in Azure Blob Storage with the given prefix.
func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
normalizedPrefix := normalizePathSeparators(prefix)
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{
Prefix: &normalizedPrefix,
})
items := make([]Item, 0)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
}
for _, blob := range resp.Segment.BlobItems {
items = append(items, Item{
Key: *blob.Name,
ModifiedTime: *blob.Properties.LastModified,
Size: *blob.Properties.ContentLength,
IsDirectory: false,
})
}
}
return items, nil
}
// Remove deletes a blob from Azure Blob Storage.
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
_, err := a.client.DeleteBlob(ctx, a.container, normalizePathSeparators(remotePath), nil)
if err != nil {
return fmt.Errorf("azure: failed to remove blob from container %s: %w", a.container, err)
}
return nil
}
// Close releases resources.
func (a *AzureStorage) Close() error {
return nil
}

251
ftp.go Normal file
View File

@@ -0,0 +1,251 @@
package go_storage
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/jlaffaye/ftp"
)
// FTPStorage implements Storage interface for FTP.
type FTPStorage struct {
host string
port string
baseDir string
client *ftp.ServerConn
}
// NewFTPStorage creates a new FTP storage instance.
func NewFTPStorage(config Config) (*FTPStorage, error) {
if err := validateConfig(config, map[string]string{
"host": config.FTPHost,
}); err != nil {
return nil, fmt.Errorf("ftp: %w", err)
}
port := config.FTPPort
if port == "" {
port = "21"
}
// Connect to FTP server
hostPort := fmt.Sprintf("%s:%s", config.FTPHost, port)
conn, err := ftp.Dial(hostPort, ftp.DialWithTimeout(30*time.Second))
if err != nil {
return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
}
// Login
if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
conn.Quit()
return nil, fmt.Errorf("ftp: login failed: %w", err)
}
return &FTPStorage{
host: config.FTPHost,
port: port,
baseDir: config.FTPDirectory,
client: conn,
}, nil
}
// Upload uploads a file via FTP.
func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
src, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("ftp: failed to open local file: %w", err)
}
defer src.Close()
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Create parent directory if needed
remoteDir := filepath.Dir(fullRemotePath)
remoteDir = normalizePathSeparators(remoteDir)
if remoteDir != "." && remoteDir != "/" {
f.ensureDirectory(remoteDir)
}
// Upload file
if err := f.client.Stor(fullRemotePath, src); err != nil {
return fmt.Errorf("ftp: failed to upload file %s: %w", fullRemotePath, err)
}
return nil
}
// Download downloads a file via FTP.
func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Get the file from FTP server
resp, err := f.client.Retr(fullRemotePath)
if err != nil {
return fmt.Errorf("ftp: failed to retrieve file %s: %w", fullRemotePath, err)
}
defer resp.Close()
// Create local file
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("ftp: failed to create local file: %w", err)
}
defer dst.Close()
// Copy data
if _, err := io.Copy(dst, resp); err != nil {
return fmt.Errorf("ftp: failed to transfer data: %w", err)
}
return nil
}
// List lists files via FTP with the given prefix.
func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
items := make([]Item, 0)
// Normalize the base directory
baseDir := normalizePathSeparators(f.baseDir)
if baseDir == "" {
baseDir = "/"
}
// Walk the directory
if err := f.walkDirectory(baseDir, prefix, &items); err != nil {
return nil, err
}
return items, nil
}
// walkDirectory recursively walks FTP directory structure.
func (f *FTPStorage) walkDirectory(dir string, prefix string, items *[]Item) error {
entries, err := f.client.List(dir)
if err != nil {
return fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
}
for _, entry := range entries {
// Get relative path
var relPath string
if f.baseDir == "" || f.baseDir == "/" {
relPath = filepath.Join(strings.TrimPrefix(dir, "/"), entry.Name)
} else {
fullPath := filepath.Join(dir, entry.Name)
relPath = strings.TrimPrefix(fullPath, f.baseDir)
relPath = strings.TrimPrefix(relPath, "/")
}
// Normalize separators for comparison
relPath = normalizePathSeparators(relPath)
// Skip if doesn't match prefix
if prefix != "" && !strings.HasPrefix(relPath, prefix) {
// For directories, we might need to walk them to find matching files
if entry.Type == ftp.EntryTypeFolder {
fullPath := filepath.Join(dir, entry.Name)
fullPath = normalizePathSeparators(fullPath)
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
// Log but don't fail on subdirectory errors
continue
}
}
continue
}
item := Item{
Key: relPath,
ModifiedTime: entry.Time,
Size: int64(entry.Size),
IsDirectory: entry.Type == ftp.EntryTypeFolder,
}
*items = append(*items, item)
// Recursively walk directories
if entry.Type == ftp.EntryTypeFolder {
fullPath := filepath.Join(dir, entry.Name)
fullPath = normalizePathSeparators(fullPath)
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
// Log but don't fail on subdirectory errors
continue
}
}
}
return nil
}
// Remove deletes a file via FTP.
func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
fullRemotePath := filepath.Join(f.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
if err := f.client.Delete(fullRemotePath); err != nil {
return fmt.Errorf("ftp: failed to remove %s: %w", fullRemotePath, err)
}
return nil
}
// Close releases resources.
func (f *FTPStorage) Close() error {
if err := f.client.Quit(); err != nil {
return fmt.Errorf("ftp: failed to close connection: %w", err)
}
return nil
}
// ensureDirectory creates a directory path if it doesn't exist.
func (f *FTPStorage) ensureDirectory(path string) {
parts := strings.Split(path, "/")
current := ""
for _, part := range parts {
if part == "" {
continue
}
if current == "" {
current = part
} else {
current = current + "/" + part
}
// Try to create directory (ignore errors if it already exists)
_ = f.client.MakeDir(current)
}
}

156
gcs.go Normal file
View File

@@ -0,0 +1,156 @@
package go_storage
import (
"context"
"fmt"
"io"
"os"
"cloud.google.com/go/storage"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)
// GCSStorage implements Storage interface for Google Cloud Storage.
type GCSStorage struct {
client *storage.Client
bucket string
}
// NewGCSStorage creates a new Google Cloud Storage instance.
func NewGCSStorage(config Config) (*GCSStorage, error) {
if err := validateConfig(config, map[string]string{
"bucket": config.GCSBucket,
}); err != nil {
return nil, fmt.Errorf("gcs: %w", err)
}
options := make([]option.ClientOption, 0)
if config.GCSEndpoint != "" {
options = append(options, option.WithEndpoint(config.GCSEndpoint))
}
if config.GCSCredentialsFile != "" {
options = append(options, option.WithCredentialsFile(config.GCSCredentialsFile))
}
client, err := storage.NewClient(context.Background(), options...)
if err != nil {
return nil, fmt.Errorf("gcs: failed to create client: %w", err)
}
return &GCSStorage{
client: client,
bucket: config.GCSBucket,
}, nil
}
// Upload uploads a file to GCS.
func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("gcs: failed to open local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("gcs: failed to close file: %v\n", err)
}
}(file)
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewWriter(ctx)
defer func(obj *storage.Writer) {
err = obj.Close()
if err != nil {
fmt.Printf("gcs: failed to close object writer: %v\n", err)
}
}(obj)
if _, err := io.Copy(obj, file); err != nil {
return fmt.Errorf("gcs: failed to write data: %w", err)
}
if err = obj.Close(); err != nil {
return fmt.Errorf("gcs: failed to finalize upload: %w", err)
}
return nil
}
// Download downloads a file from GCS.
func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("gcs: failed to create local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("gcs: failed to close file: %v\n", err)
}
}(file)
obj, err := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewReader(ctx)
if err != nil {
return fmt.Errorf("gcs: failed to create reader: %w", err)
}
defer func(obj *storage.Reader) {
err = obj.Close()
if err != nil {
fmt.Printf("gcs: failed to close object reader: %v\n", err)
}
}(obj)
if _, err = io.Copy(file, obj); err != nil {
return fmt.Errorf("gcs: failed to read data: %w", err)
}
return nil
}
// List lists objects in GCS with the given prefix.
func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
items := make([]Item, 0)
query := &storage.Query{
Prefix: normalizePathSeparators(prefix),
}
it := g.client.Bucket(g.bucket).Objects(ctx, query)
for {
attrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return nil, fmt.Errorf("gcs: failed to iterate objects: %w", err)
}
items = append(items, Item{
Key: attrs.Name,
ModifiedTime: attrs.Updated,
Size: attrs.Size,
IsDirectory: false,
})
}
return items, nil
}
// Remove deletes an object from GCS.
func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath))
if err := obj.Delete(ctx); err != nil {
return fmt.Errorf("gcs: failed to remove object from bucket %s: %w", g.bucket, err)
}
return nil
}
// Close releases resources.
func (g *GCSStorage) Close() error {
return g.client.Close()
}

67
go.mod
View File

@@ -1,22 +1,67 @@
module github.com/jkaninda/go-storage
go 1.21.0
go 1.24.0
require (
github.com/spf13/pflag v1.0.5
cloud.google.com/go/storage v1.60.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
github.com/aws/aws-sdk-go v1.55.7
github.com/bramvdbogaerde/go-scp v1.5.0
github.com/jlaffaye/ftp v0.2.0
github.com/pkg/sftp v1.13.10
golang.org/x/crypto v0.47.0
google.golang.org/api v0.267.0
)
require (
github.com/aws/aws-sdk-go v1.55.3 // indirect
github.com/bramvdbogaerde/go-scp v1.5.0 // indirect
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.18.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hpcloud/tail v1.0.0 // indirect
github.com/jlaffaye/ftp v0.2.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
golang.org/x/net v0.49.0 // indirect
golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/grpc v1.78.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)

198
go.sum
View File

@@ -1,68 +1,170 @@
github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0=
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

348
local.go Normal file
View File

@@ -0,0 +1,348 @@
package go_storage
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// LocalStorage implements the Storage interface using the local filesystem.
// It is safe for concurrent use as it relies on filesystem operations which
// are handled by the OS.
type LocalStorage struct {
basePath string
}
// NewLocalStorage creates a new LocalStorage instance with the given base directory.
// The base directory will be created if it doesn't exist.
func NewLocalStorage(basePath string) (*LocalStorage, error) {
absPath, err := filepath.Abs(basePath)
if err != nil {
return nil, fmt.Errorf("failed to resolve absolute path: %w", err)
}
if err := os.MkdirAll(absPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create base directory: %w", err)
}
return &LocalStorage{
basePath: absPath,
}, nil
}
// resolvePath safely resolves a remote path to an absolute local path,
// ensuring it stays within the base directory.
func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
// Clean the path to remove any .. or . components
cleanPath := filepath.Clean(remotePath)
// Remove leading slashes to make it relative
cleanPath = strings.TrimPrefix(cleanPath, string(filepath.Separator))
// Join with base path
fullPath := filepath.Join(s.basePath, cleanPath)
// Verify the resolved path is still within basePath
if !strings.HasPrefix(fullPath, s.basePath) {
return "", fmt.Errorf("path traversal attempt detected: %s", remotePath)
}
return fullPath, nil
}
// Upload uploads a local file to the storage with the given target name.
func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
targetPath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Open source file
srcFile, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("failed to open source file: %w", err)
}
defer srcFile.Close()
// Create parent directories if needed
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return fmt.Errorf("failed to create parent directories: %w", err)
}
// Create destination file
dstFile, err := os.Create(targetPath)
if err != nil {
return fmt.Errorf("failed to create destination file: %w", err)
}
defer dstFile.Close()
// Copy with context cancellation checks
buf := make([]byte, 32*1024) // 32KB buffer
for {
select {
case <-ctx.Done():
// Clean up partial file on cancellation
os.Remove(targetPath)
return ctx.Err()
default:
}
n, readErr := srcFile.Read(buf)
if n > 0 {
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
os.Remove(targetPath)
return fmt.Errorf("failed to write to destination: %w", writeErr)
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
os.Remove(targetPath)
return fmt.Errorf("failed to read source file: %w", readErr)
}
}
return nil
}
// Download downloads a file from storage to a local path.
func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
sourcePath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Open source file
srcFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("failed to open remote file: %w", err)
}
defer srcFile.Close()
// Create parent directories for local path if needed
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
return fmt.Errorf("failed to create local parent directories: %w", err)
}
// Create destination file
dstFile, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("failed to create local file: %w", err)
}
defer dstFile.Close()
// Copy with context cancellation checks
buf := make([]byte, 32*1024) // 32KB buffer
for {
select {
case <-ctx.Done():
// Clean up partial file on cancellation
os.Remove(localPath)
return ctx.Err()
default:
}
n, readErr := srcFile.Read(buf)
if n > 0 {
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
os.Remove(localPath)
return fmt.Errorf("failed to write to local file: %w", writeErr)
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
os.Remove(localPath)
return fmt.Errorf("failed to read remote file: %w", readErr)
}
}
return nil
}
// List returns all items in the storage that match the given prefix.
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
var items []Item
// Clean and resolve the prefix path
searchPath := s.basePath
cleanPrefix := strings.TrimPrefix(filepath.Clean(prefix), string(filepath.Separator))
if cleanPrefix != "" && cleanPrefix != "." {
searchPath = filepath.Join(s.basePath, cleanPrefix)
}
// Check if the search path exists
info, err := os.Stat(searchPath)
if os.IsNotExist(err) {
// If the exact path doesn't exist, try to find files with this prefix
// by walking the parent directory
parentDir := filepath.Dir(searchPath)
baseName := filepath.Base(searchPath)
if _, err := os.Stat(parentDir); os.IsNotExist(err) {
return items, nil // Return empty list if parent doesn't exist
}
entries, err := os.ReadDir(parentDir)
if err != nil {
return nil, fmt.Errorf("failed to read directory: %w", err)
}
for _, entry := range entries {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if strings.HasPrefix(entry.Name(), baseName) {
fullPath := filepath.Join(parentDir, entry.Name())
item, err := s.fileInfoToItem(fullPath)
if err != nil {
continue // Skip files we can't stat
}
items = append(items, item)
}
}
return items, nil
}
if err != nil {
return nil, fmt.Errorf("failed to stat path: %w", err)
}
// If it's a file, return just that file
if !info.IsDir() {
item, err := s.fileInfoToItem(searchPath)
if err != nil {
return nil, err
}
return []Item{item}, nil
}
// Walk the directory
err = filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err != nil {
return nil // Skip entries with errors
}
item, err := s.fileInfoToItem(path)
if err != nil {
return nil // Skip files we can't stat
}
items = append(items, item)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to walk directory: %w", err)
}
return items, nil
}
// fileInfoToItem converts a file path to an Item.
func (s *LocalStorage) fileInfoToItem(path string) (Item, error) {
info, err := os.Stat(path)
if err != nil {
return Item{}, err
}
// Get relative path from base
relPath, err := filepath.Rel(s.basePath, path)
if err != nil {
relPath = path
}
// Normalize to forward slashes for consistency
relPath = filepath.ToSlash(relPath)
size := info.Size()
if info.IsDir() {
size = 0
}
return Item{
Key: relPath,
ModifiedTime: info.ModTime(),
Size: size,
IsDirectory: info.IsDir(),
}, nil
}
// Remove deletes a file from storage.
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
targetPath, err := s.resolvePath(remotePath)
if err != nil {
return err
}
// Check if file exists
info, err := os.Stat(targetPath)
if os.IsNotExist(err) {
return fmt.Errorf("file not found: %s", remotePath)
}
if err != nil {
return fmt.Errorf("failed to stat file: %w", err)
}
// Remove file or directory
if info.IsDir() {
if err := os.RemoveAll(targetPath); err != nil {
return fmt.Errorf("failed to remove directory: %w", err)
}
} else {
if err := os.Remove(targetPath); err != nil {
return fmt.Errorf("failed to remove file: %w", err)
}
}
return nil
}
// Close releases any resources held by the storage implementation.
// For LocalStorage, this is a no-op as there are no persistent resources.
func (s *LocalStorage) Close() error {
return nil
}
// BasePath returns the base directory path of the storage.
func (s *LocalStorage) BasePath() string {
return s.basePath
}

161
pkg/azure/azure.go Normal file
View File

@@ -0,0 +1,161 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package azure
import (
"context"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/jkaninda/go-storage/pkg"
"io"
"log"
"os"
"path/filepath"
)
type azureStorage struct {
*pkg.Backend
client *azblob.Client
containerName string
}
type Config struct {
AccountName string
AccountKey string
ContainerName string
LocalPath string
RemotePath string
}
// createClient creates FTP Client
func createClient(conf Config) (*azblob.Client, error) {
// Create the service URL
credential, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
if err != nil {
return nil, fmt.Errorf("failed to create service URL")
}
serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName)
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil)
if err != nil {
return nil, fmt.Errorf("failed to create client")
}
return client, nil
}
// NewStorage creates new Storage
func NewStorage(conf Config) (pkg.Storage, error) {
client, err := createClient(conf)
if err != nil {
return nil, err
}
return &azureStorage{
client: client,
containerName: conf.ContainerName,
Backend: &pkg.Backend{
RemotePath: conf.RemotePath,
LocalPath: conf.LocalPath,
},
}, nil
}
// Copy copies file to Azure Blob Storage
func (s azureStorage) Copy(fileName string) error {
filePath := filepath.Join(s.LocalPath, fileName)
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", fileName, err)
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
return
}
}(file)
blobName := filepath.Join(s.RemotePath, fileName)
// Get the container client
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
// Get the blob client
blobClient := containerClient.NewBlockBlobClient(blobName)
// Upload the file
_, err = blobClient.UploadFile(context.Background(), file, nil)
if err != nil {
log.Fatalf("Failed to upload file: %v", err)
}
return nil
}
// CopyFrom copies a file from Azure Blob Storage to local storage
func (s azureStorage) CopyFrom(blobName string) error {
// Get the container client
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
// Get the blob client
blobClient := containerClient.NewBlockBlobClient(filepath.Join(s.RemotePath, blobName))
// Download the blob
downloadResponse, err := blobClient.DownloadStream(context.Background(), nil)
if err != nil {
return err
}
// Create the file to save the blob data
downloadFile, err := os.Create(filepath.Join(s.LocalPath, blobName))
if err != nil {
return err
}
defer func(downloadFile *os.File) {
err := downloadFile.Close()
if err != nil {
log.Fatalf("Failed to close file: %v", err)
}
}(downloadFile)
// Write the blob data to the file
_, err = io.Copy(downloadFile, downloadResponse.Body)
if err != nil {
log.Fatalf("Failed to write blob to file: %v", err)
}
return nil
}
// Prune deletes old backup created more than specified days
func (s azureStorage) Prune(retentionDays int) error {
fmt.Println("Deleting old backup from a remote server is not implemented yet")
return nil
}
// Name returns the storage name
func (s azureStorage) Name() string {
return "azure"
}

View File

@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package ftp
import (
@@ -20,14 +44,14 @@ type Config struct {
Host string
User string
Password string
Port string
Port int
LocalPath string
RemotePath string
}
// createClient creates FTP Client
func createClient(conf Config) (*ftp.ServerConn, error) {
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%d", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
if err != nil {
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
}
@@ -58,14 +82,24 @@ func NewStorage(conf Config) (pkg.Storage, error) {
// Copy copies file to the remote server
func (s ftpStorage) Copy(fileName string) error {
ftpClient := s.client
defer ftpClient.Quit()
defer func(ftpClient *ftp.ServerConn) {
err := ftpClient.Quit()
if err != nil {
return
}
}(ftpClient)
filePath := filepath.Join(s.LocalPath, fileName)
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", fileName, err)
}
defer file.Close()
defer func(file *os.File) {
err := file.Close()
if err != nil {
return
}
}(file)
remoteFilePath := filepath.Join(s.RemotePath, fileName)
err = ftpClient.Stor(remoteFilePath, file)
@@ -80,21 +114,36 @@ func (s ftpStorage) Copy(fileName string) error {
func (s ftpStorage) CopyFrom(fileName string) error {
ftpClient := s.client
defer ftpClient.Quit()
defer func(ftpClient *ftp.ServerConn) {
err := ftpClient.Quit()
if err != nil {
return
}
}(ftpClient)
remoteFilePath := filepath.Join(s.RemotePath, fileName)
r, err := ftpClient.Retr(remoteFilePath)
if err != nil {
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
}
defer r.Close()
defer func(r *ftp.Response) {
err := r.Close()
if err != nil {
return
}
}(r)
localFilePath := filepath.Join(s.LocalPath, fileName)
outFile, err := os.Create(localFilePath)
if err != nil {
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
}
defer outFile.Close()
defer func(outFile *os.File) {
err := outFile.Close()
if err != nil {
return
}
}(outFile)
_, err = io.Copy(outFile, r)
if err != nil {

View File

@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package local
import (
@@ -92,7 +116,12 @@ func copyFile(src, dst string) error {
if err != nil {
return err
}
defer in.Close()
defer func(in *os.File) {
err := in.Close()
if err != nil {
return
}
}(in)
out, err := os.Create(dst)
if err != nil {
@@ -101,7 +130,10 @@ func copyFile(src, dst string) error {
_, err = io.Copy(out, in)
if err != nil {
out.Close()
err := out.Close()
if err != nil {
return err
}
return err
}
return out.Close()

View File

@@ -45,7 +45,12 @@ func createFile(fileName, content string) ([]byte, error) {
fmt.Println("Error creating file:", err)
return nil, err
}
defer file.Close()
defer func(file *os.File) {
err := file.Close()
if err != nil {
return
}
}(file)
// Write the message to the file
_, err = file.WriteString(content)

View File

@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package s3
import (
@@ -70,7 +94,12 @@ func (s s3Storage) Copy(fileName string) error {
if err != nil {
return err
}
defer file.Close()
defer func(file *os.File) {
err := file.Close()
if err != nil {
return
}
}(file)
fileInfo, err := file.Stat()
if err != nil {
@@ -78,7 +107,10 @@ func (s s3Storage) Copy(fileName string) error {
}
objectKey := filepath.Join(s.RemotePath, fileName)
buffer := make([]byte, fileInfo.Size())
file.Read(buffer)
_, err = file.Read(buffer)
if err != nil {
return err
}
fileBytes := bytes.NewReader(buffer)
fileType := http.DetectContentType(buffer)
@@ -102,7 +134,13 @@ func (s s3Storage) CopyFrom(fileName string) error {
if err != nil {
return err
}
defer file.Close()
defer func(file *os.File) {
err := file.Close()
if err != nil {
fmt.Printf("Error closing file: %v\n", err)
return
}
}(file)
objectKey := filepath.Join(s.RemotePath, fileName)
@@ -140,16 +178,16 @@ func (s s3Storage) Prune(retentionDays int) error {
Key: object.Key,
})
if err != nil {
fmt.Printf("failed to delete object %s: %v", *object.Key, err)
fmt.Printf("failed to delete object %s: %v\n", *object.Key, err)
} else {
fmt.Printf("Deleted object %s", *object.Key)
fmt.Printf("Deleted object %s\n", *object.Key)
}
}
}
return !lastPage
})
if err != nil {
return fmt.Errorf("failed to list objects: %v", err)
return fmt.Errorf("failed to list objects: %v\n", err)
}
return nil

View File

@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package ssh
import (
@@ -22,7 +46,7 @@ type Config struct {
Host string
User string
Password string
Port string
Port int
IdentifyFile string
LocalPath string
RemotePath string
@@ -30,15 +54,15 @@ type Config struct {
// createClient creates SSH Client
func createClient(conf Config) (scp.Client, error) {
if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) {
if _, err := os.Stat(conf.IdentifyFile); !os.IsNotExist(err) {
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
} else {
if conf.Password == "" {
return scp.Client{}, errors.New("ssh password required")
}
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
}
}
@@ -97,7 +121,12 @@ func (s sshStorage) CopyFrom(fileName string) error {
if err != nil {
return errors.New("couldn't open the output file")
}
defer file.Close()
defer func(file *os.File) {
err := file.Close()
if err != nil {
return
}
}(file)
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))

1
pkg/ssh/ssh_test.go Normal file
View File

@@ -0,0 +1 @@
package ssh

View File

@@ -1,3 +1,27 @@
/*
MIT License
Copyright (c) 2023 Jonas Kaninda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package pkg
type Storage interface {
@@ -7,8 +31,8 @@ type Storage interface {
Name() string
}
type Backend struct {
//Local Path
// Local Path
LocalPath string
//Remote path or Destination path
// Remote path or Destination path
RemotePath string
}

187
s3.go Normal file
View File

@@ -0,0 +1,187 @@
package go_storage
import (
"context"
"fmt"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
// S3Storage implements Storage interface for Amazon S3.
type S3Storage struct {
session *session.Session
bucket string
region string
}
// NewS3Storage creates a new S3 storage instance.
func NewS3Storage(config Config) (*S3Storage, error) {
if err := validateConfig(config, map[string]string{
"bucket": config.S3Bucket,
}); err != nil {
return nil, fmt.Errorf("s3: %w", err)
}
awsConfig := aws.NewConfig()
if config.S3Region != "" {
awsConfig = awsConfig.WithRegion(config.S3Region)
}
if config.S3KeyID != "" && config.S3Secret != "" {
awsConfig = awsConfig.WithCredentials(
credentials.NewStaticCredentials(config.S3KeyID, config.S3Secret, ""),
)
}
if config.S3Endpoint != "" {
awsConfig = awsConfig.WithEndpoint(config.S3Endpoint)
}
if config.S3ForcePath {
awsConfig = awsConfig.WithS3ForcePathStyle(true)
}
if config.S3DisableTLS {
awsConfig = awsConfig.WithDisableSSL(true)
}
sessionOpts := session.Options{
Config: *awsConfig,
SharedConfigState: session.SharedConfigEnable,
}
if config.S3Profile != "" {
sessionOpts.Profile = config.S3Profile
}
sess, err := session.NewSessionWithOptions(sessionOpts)
if err != nil {
return nil, fmt.Errorf("s3: failed to create session: %w", err)
}
return &S3Storage{
session: sess,
bucket: config.S3Bucket,
region: config.S3Region,
}, nil
}
// Upload uploads a file to S3.
func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("s3: failed to open local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("s3: warning: failed to close file: %v\n", err)
}
}(file)
uploader := s3manager.NewUploader(s.session)
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
Body: file,
})
if err != nil {
return fmt.Errorf("s3: failed to upload to bucket %s: %w", s.bucket, err)
}
return nil
}
// Download downloads a file from S3.
func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("s3: failed to create local file: %w", err)
}
defer func(file *os.File) {
err = file.Close()
if err != nil {
fmt.Printf("s3: warning: failed to close file: %v\n", err)
}
}(file)
downloader := s3manager.NewDownloader(s.session)
_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
})
if err != nil {
return fmt.Errorf("s3: failed to download from bucket %s: %w", s.bucket, err)
}
return nil
}
// List lists objects in S3 with the given prefix.
func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
svc := s3.New(s.session)
items := make([]Item, 0)
var continuationToken *string
for {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(s.bucket),
Prefix: aws.String(normalizePathSeparators(prefix)),
ContinuationToken: continuationToken,
}
resp, err := svc.ListObjectsV2WithContext(ctx, input)
if err != nil {
return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
}
for _, obj := range resp.Contents {
items = append(items, Item{
Key: *obj.Key,
ModifiedTime: *obj.LastModified,
Size: *obj.Size,
IsDirectory: false,
})
}
if resp.IsTruncated == nil || !*resp.IsTruncated {
break
}
continuationToken = resp.NextContinuationToken
}
return items, nil
}
// Remove deletes an object from S3.
func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
svc := s3.New(s.session)
_, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(normalizePathSeparators(remotePath)),
})
if err != nil {
return fmt.Errorf("s3: failed to remove object from bucket %s: %w", s.bucket, err)
}
return nil
}
// Close releases resources.
func (s *S3Storage) Close() error {
return nil
}

350
sftp.go Normal file
View File

@@ -0,0 +1,350 @@
package go_storage
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/crypto/ssh/knownhosts"
)
// SFTPStorage implements Storage interface for SFTP.
type SFTPStorage struct {
host string
baseDir string
sshClient *ssh.Client
client *sftp.Client
}
// NewSFTPStorage creates a new SFTP storage instance.
func NewSFTPStorage(config Config) (*SFTPStorage, error) {
if err := validateConfig(config, map[string]string{
"host": config.SFTPHost,
}); err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
port := config.SFTPPort
if port == "" {
port = "22"
}
username := config.SFTPUsername
if username == "" {
currentUser, err := user.Current()
if err != nil {
return nil, fmt.Errorf("sftp: failed to get current user: %w", err)
}
username = currentUser.Username
}
password := config.SFTPPassword
if password == "" {
password = os.Getenv("PGBK_SSH_PASS")
}
// Build authentication methods
authMethods := make([]ssh.AuthMethod, 0)
// Password authentication (if no identity file is specified)
if config.SFTPIdentityFile == "" && password != "" {
authMethods = append(authMethods, ssh.Password(password))
}
// Public key authentication
signers, err := getSSHSigners(config.SFTPIdentityFile, password)
if err != nil {
return nil, fmt.Errorf("sftp: %w", err)
}
if len(signers) > 0 {
authMethods = append(authMethods, ssh.PublicKeys(signers...))
}
// Build SSH client config
sshConfig := &ssh.ClientConfig{
User: username,
Auth: authMethods,
HostKeyCallback: getHostKeyCallback(config.SFTPIgnoreKnownHosts),
}
// Connect to SSH server
hostPort := fmt.Sprintf("%s:%s", config.SFTPHost, port)
sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
if err != nil {
return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
}
// Open SFTP session
sftpClient, err := sftp.NewClient(sshClient)
if err != nil {
sshClient.Close()
return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
}
baseDir := config.SFTPDirectory
if baseDir == "" {
wd, err := sftpClient.Getwd()
if err != nil {
sftpClient.Close()
sshClient.Close()
return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
}
baseDir = wd
}
return &SFTPStorage{
host: config.SFTPHost,
baseDir: baseDir,
sshClient: sshClient,
client: sftpClient,
}, nil
}
// Upload uploads a file via SFTP.
func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
src, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("sftp: failed to open local file: %w", err)
}
defer src.Close()
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
// Create parent directory if needed
remoteDir := filepath.Dir(fullRemotePath)
remoteDir = normalizePathSeparators(remoteDir)
if remoteDir != "." && remoteDir != "/" {
if err := s.client.MkdirAll(remoteDir); err != nil {
return fmt.Errorf("sftp: failed to create directory %s: %w", remoteDir, err)
}
}
dst, err := s.client.Create(fullRemotePath)
if err != nil {
return fmt.Errorf("sftp: failed to create remote file %s: %w", fullRemotePath, err)
}
defer dst.Close()
if _, err := io.Copy(dst, src); err != nil {
return fmt.Errorf("sftp: failed to transfer data: %w", err)
}
return nil
}
// Download downloads a file via SFTP.
func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("sftp: failed to create local file: %w", err)
}
defer dst.Close()
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
src, err := s.client.Open(fullRemotePath)
if err != nil {
return fmt.Errorf("sftp: failed to open remote file %s: %w", fullRemotePath, err)
}
defer src.Close()
if _, err := io.Copy(dst, src); err != nil {
return fmt.Errorf("sftp: failed to transfer data: %w", err)
}
return nil
}
// List lists files via SFTP with the given prefix.
func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
items := make([]Item, 0)
baseDir := normalizePathSeparators(s.baseDir)
walker := s.client.Walk(baseDir)
for walker.Step() {
if err := walker.Err(); err != nil {
return nil, fmt.Errorf("sftp: walk error: %w", err)
}
// Get relative path
path := walker.Path()
relPath, err := filepath.Rel(baseDir, path)
if err != nil {
continue
}
// Normalize separators for comparison
relPath = normalizePathSeparators(relPath)
if !strings.HasPrefix(relPath, prefix) {
continue
}
stat := walker.Stat()
items = append(items, Item{
Key: relPath,
ModifiedTime: stat.ModTime(),
Size: stat.Size(),
IsDirectory: stat.IsDir(),
})
}
return items, nil
}
// Remove deletes a file via SFTP.
func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
fullRemotePath := filepath.Join(s.baseDir, remotePath)
fullRemotePath = normalizePathSeparators(fullRemotePath)
if err := s.client.Remove(fullRemotePath); err != nil {
return fmt.Errorf("sftp: failed to remove %s: %w", fullRemotePath, err)
}
return nil
}
// Close releases resources.
func (s *SFTPStorage) Close() error {
if err := s.client.Close(); err != nil {
s.sshClient.Close()
return err
}
return s.sshClient.Close()
}
// getSSHSigners returns SSH signers from identity file and SSH agent.
func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
signers := make([]ssh.Signer, 0)
// Load identity file if provided
if identityFile != "" {
path, err := expandHomeDir(identityFile)
if err != nil {
return nil, fmt.Errorf("failed to expand identity file path: %w", err)
}
keyData, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read identity file %s: %w", path, err)
}
signer, err := ssh.ParsePrivateKey(keyData)
if err != nil {
var passErr *ssh.PassphraseMissingError
if errors.As(err, &passErr) {
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
if err != nil {
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
}
} else {
return nil, fmt.Errorf("failed to parse identity file: %w", err)
}
}
signers = append(signers, signer)
}
// Try to get keys from SSH agent
socket := os.Getenv("SSH_AUTH_SOCK")
if socket != "" {
conn, err := net.Dial("unix", socket)
if err == nil {
agentClient := agent.NewClient(conn)
agentSigners, err := agentClient.Signers()
if err == nil {
signers = append(signers, agentSigners...)
}
}
}
return signers, nil
}
// getHostKeyCallback returns appropriate host key callback.
func getHostKeyCallback(ignoreHostKey bool) ssh.HostKeyCallback {
if ignoreHostKey {
return ssh.InsecureIgnoreHostKey()
}
knownHostsFiles := make([]string, 0)
for _, p := range []string{"/etc/ssh/ssh_known_hosts", "~/.ssh/known_hosts"} {
path, err := expandHomeDir(p)
if err != nil {
continue
}
if _, err := os.Stat(path); err == nil {
knownHostsFiles = append(knownHostsFiles, path)
}
}
if len(knownHostsFiles) == 0 {
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return fmt.Errorf("no known_hosts files found for host key verification")
}
}
callback, err := knownhosts.New(knownHostsFiles...)
if err != nil {
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return fmt.Errorf("failed to load known_hosts: %w", err)
}
}
return callback
}
// expandHomeDir expands ~ in file paths to home directory.
func expandHomeDir(path string) (string, error) {
if !strings.HasPrefix(path, "~") {
return filepath.Clean(path), nil
}
parts := strings.SplitN(path, "/", 2)
username := strings.TrimPrefix(parts[0], "~")
var homeDir string
var err error
if username == "" {
homeDir, err = os.UserHomeDir()
if err != nil || homeDir == "" {
currentUser, err := user.Current()
if err != nil {
return "", fmt.Errorf("failed to get home directory: %w", err)
}
homeDir = currentUser.HomeDir
}
} else {
userInfo, err := user.Lookup(username)
if err != nil {
return "", fmt.Errorf("failed to lookup user %s: %w", username, err)
}
homeDir = userInfo.HomeDir
}
if homeDir == "" {
return "", fmt.Errorf("empty home directory")
}
if len(parts) == 1 {
return homeDir, nil
}
return filepath.Join(homeDir, parts[1]), nil
}

140
storage.go Normal file
View File

@@ -0,0 +1,140 @@
package go_storage
import (
"context"
"fmt"
"os"
"strings"
"time"
)
// Storage defines the interface for remote file storage operations.
// All implementations must be safe for concurrent use.
type Storage interface {
// Upload uploads a local file to the remote storage with the given target name.
Upload(ctx context.Context, localPath string, remotePath string) error
// Download downloads a file from remote storage to a local path.
Download(ctx context.Context, remotePath string, localPath string) error
// List returns all items in the storage that match the given prefix.
List(ctx context.Context, prefix string) ([]Item, error)
// Remove deletes a file from remote storage.
Remove(ctx context.Context, remotePath string) error
// Close releases any resources held by the storage implementation.
Close() error
}
// Item represents a file or directory in remote storage.
type Item struct {
Key string // The path/key of the item
ModifiedTime time.Time // Last modification time
Size int64 // Size in bytes (0 for directories)
IsDirectory bool // Whether this item is a directory
}
// Config holds configuration options for creating storage instances.
type Config struct {
// S3 configuration
S3Region string
S3Bucket string
S3Profile string
S3KeyID string
S3Secret string
S3Endpoint string
S3ForcePath bool
S3DisableTLS bool
// B2 configuration
B2KeyID string
B2AppKey string
B2Bucket string
B2ConcurrentConnections int
B2ForcePath bool
// Google Cloud Storage configuration
GCSBucket string
GCSEndpoint string
GCSCredentialsFile string
// Azure configuration
AzureAccount string
AzureKey string
AzureContainer string
AzureEndpoint string
// SFTP configuration
SFTPHost string
SFTPPort string
SFTPUsername string
SFTPPassword string
SFTPDirectory string
SFTPIdentityFile string
SFTPIgnoreKnownHosts bool
// FTP configuration
FTPHost string
FTPPort string
FTPUsername string
FTPPassword string
FTPDirectory string
}
// StorageType represents the type of storage backend.
type StorageType string
const (
StorageTypeS3 StorageType = "s3"
StorageTypeGCS StorageType = "gcs"
StorageTypeAzure StorageType = "azure"
StorageTypeSFTP StorageType = "sftp"
StorageTypeFTP StorageType = "ftp"
StorageTypeLocal StorageType = "local"
)
// NewStorage creates a new Storage instance based on the specified type and configuration.
func NewStorage(storageType StorageType, config Config) (Storage, error) {
switch storageType {
case StorageTypeS3:
return NewS3Storage(config)
case StorageTypeGCS:
return NewGCSStorage(config)
case StorageTypeAzure:
return NewAzureStorage(config)
case StorageTypeSFTP:
return NewSFTPStorage(config)
case StorageTypeFTP:
return NewFTPStorage(config)
case StorageTypeLocal:
return NewLocalStorage("/backups")
default:
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
}
}
// normalizePathSeparators converts OS-specific path separators to forward slashes.
// This is necessary for cloud storage services which expect forward slashes.
func normalizePathSeparators(path string) string {
if os.PathSeparator == '/' {
return path
}
return strings.ReplaceAll(path, string(os.PathSeparator), "/")
}
// validateConfig checks if the required configuration fields are present.
func validateConfig(_ Config, requiredFields map[string]string) error {
var missing []string
for field, value := range requiredFields {
if value == "" {
missing = append(missing, field)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", "))
}
return nil
}

41
storage_test.go Normal file
View File

@@ -0,0 +1,41 @@
package go_storage
import (
"context"
"testing"
"time"
)
// Test helpers
func TestStorageInterface(t *testing.T) {
var _ Storage = (*S3Storage)(nil)
var _ Storage = (*GCSStorage)(nil)
var _ Storage = (*AzureStorage)(nil)
var _ Storage = (*SFTPStorage)(nil)
}
func TestContextTimeout(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
config := Config{
S3Region: "us-east-1",
S3Bucket: "test-bucket",
}
s3, err := NewStorage(StorageTypeS3, config)
if err != nil {
t.Fatalf("Failed to create storage: %v", err)
}
defer func(s3 Storage) {
err = s3.Close()
if err != nil {
t.Logf("Failed to close storage: %v", err)
}
}(s3)
err = s3.Upload(ctx, "test.txt", "remote-test.txt")
if err != nil {
t.Logf("Upload failed (expected in test): %v", err)
}
}