mirror of
https://github.com/jkaninda/go-storage.git
synced 2026-03-11 12:09:06 +01:00
Compare commits
24 Commits
v0.1.1
...
v0.2.0-rc1
| Author | SHA1 | Date | |
|---|---|---|---|
| fdbf0cd233 | |||
|
|
e5e68bdadf | ||
|
|
652d00ce3b | ||
|
|
c39d828fe6 | ||
|
|
b9d06c6529 | ||
|
|
373e8ada7a | ||
|
|
da0fc37905 | ||
|
|
cd01e0007c | ||
|
|
bcdcffed55 | ||
|
|
624c986a23 | ||
|
|
a06c6acdb8 | ||
|
|
538df008f7 | ||
|
|
10e9be1f49 | ||
|
|
367de149f2 | ||
| 1e36fb1884 | |||
|
|
da738264ab | ||
| 085bdf468f | |||
|
|
0a70da956f | ||
|
|
793b04340e | ||
|
|
1b147ebea5 | ||
|
|
2bcfd3aacf | ||
|
|
845e0cc43b | ||
|
|
d314c3854b | ||
|
|
2339db673d |
6
.github/dependabot.yml
vendored
Normal file
6
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: gomod
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
23
.github/workflows/lint.yml
vendored
Normal file
23
.github/workflows/lint.yml
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
name: Lint
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Run on Ubuntu
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Clone the code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '~1.24'
|
||||
|
||||
- name: Run linter
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
with:
|
||||
version: v2.1.2
|
||||
47
.golangci.yml
Normal file
47
.golangci.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
version: "2"
|
||||
run:
|
||||
allow-parallel-runners: true
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- copyloopvar
|
||||
- dupl
|
||||
- errcheck
|
||||
- ginkgolinter
|
||||
- goconst
|
||||
- gocyclo
|
||||
- govet
|
||||
- ineffassign
|
||||
#- lll
|
||||
- misspell
|
||||
- nakedret
|
||||
- prealloc
|
||||
- revive
|
||||
- staticcheck
|
||||
- unconvert
|
||||
- unparam
|
||||
- unused
|
||||
settings:
|
||||
revive:
|
||||
rules:
|
||||
- name: comment-spacings
|
||||
exclusions:
|
||||
generated: lax
|
||||
rules:
|
||||
- linters:
|
||||
- dupl
|
||||
- lll
|
||||
path: /*
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
30
README.md
30
README.md
@@ -6,6 +6,7 @@ Supported storage:
|
||||
- S3
|
||||
- SSH
|
||||
- FTP
|
||||
- Azure Blob
|
||||
|
||||
```go
|
||||
go get github.com/jkaninda/go-storage
|
||||
@@ -55,7 +56,7 @@ if err != nil {
|
||||
```go
|
||||
sshStorage, err := ssh.NewStorage(ssh.Config{
|
||||
Host: "",
|
||||
Port: "",
|
||||
Port: 22,
|
||||
User: "",
|
||||
Password: "",
|
||||
RemotePath: "",
|
||||
@@ -79,7 +80,7 @@ log.Fatalf("Error copying file, error %v", err)
|
||||
```go
|
||||
ftpStorage, err := ftp.NewStorage(ftp.Config{
|
||||
Host: "",
|
||||
Port: "",
|
||||
Port: 21,
|
||||
User: "",
|
||||
Password: "",
|
||||
RemotePath: "",
|
||||
@@ -97,4 +98,29 @@ err = ftpStorage.CopyFrom(finalFileName)
|
||||
if err != nil {
|
||||
log.Fatalf("Error copying file, error %v", err)
|
||||
}
|
||||
```
|
||||
|
||||
### Azure Blob storage
|
||||
|
||||
```go
|
||||
azureStorage, err := azure.NewStorage(azure.Config{
|
||||
ContainerName: '',
|
||||
AccountName: '',
|
||||
AccountKey: '',
|
||||
RemotePath: '',
|
||||
LocalPath: '',
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal("Error creating Azure Blob storage storage: %s", err)
|
||||
}
|
||||
err = azureStorage.Copy(finalFileName)
|
||||
if err != nil {
|
||||
log.Fatal("Error copying file: %s", err)
|
||||
}
|
||||
|
||||
// Download file from Azure Blob remote server
|
||||
err = azureStorage.CopyFrom(finalFileName)
|
||||
if err != nil {
|
||||
log.Fatalf("Error copying file, error %v", err)
|
||||
}
|
||||
```
|
||||
155
azure.go
Normal file
155
azure.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
)
|
||||
|
||||
// AzureStorage implements Storage interface for Azure Blob Storage.
|
||||
type AzureStorage struct {
|
||||
client *azblob.Client
|
||||
container string
|
||||
}
|
||||
|
||||
// NewAzureStorage creates a new Azure Blob Storage instance.
|
||||
func NewAzureStorage(config Config) (*AzureStorage, error) {
|
||||
if err := validateConfig(config, map[string]string{
|
||||
"container": config.AzureContainer,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("azure: %w", err)
|
||||
}
|
||||
|
||||
// Try to get credentials from environment if not provided
|
||||
account := config.AzureAccount
|
||||
if account == "" {
|
||||
account = os.Getenv("AZURE_STORAGE_ACCOUNT")
|
||||
}
|
||||
|
||||
key := config.AzureKey
|
||||
if key == "" {
|
||||
key = os.Getenv("AZURE_STORAGE_KEY")
|
||||
}
|
||||
|
||||
endpoint := config.AzureEndpoint
|
||||
if endpoint == "" {
|
||||
endpoint = "blob.core.windows.net"
|
||||
}
|
||||
|
||||
var client *azblob.Client
|
||||
var err error
|
||||
|
||||
if account == "" {
|
||||
// Anonymous access
|
||||
client, err = azblob.NewClientWithNoCredential(endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azure: failed to create anonymous client: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Authenticated access
|
||||
credential, err := azblob.NewSharedKeyCredential(account, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azure: failed to create credentials: %w", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://%s.%s", account, endpoint)
|
||||
|
||||
client, err = azblob.NewClientWithSharedKeyCredential(url, credential, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azure: failed to create client: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &AzureStorage{
|
||||
client: client,
|
||||
container: config.AzureContainer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload uploads a file to Azure Blob Storage.
|
||||
func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure: failed to open local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("azure: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
_, err = a.client.UploadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure: failed to upload to container %s: %w", a.container, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file from Azure Blob Storage.
|
||||
func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
file, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure: failed to create local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("azure: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
_, err = a.client.DownloadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure: failed to download from container %s: %w", a.container, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists blobs in Azure Blob Storage with the given prefix.
|
||||
func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
normalizedPrefix := normalizePathSeparators(prefix)
|
||||
|
||||
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{
|
||||
Prefix: &normalizedPrefix,
|
||||
})
|
||||
|
||||
items := make([]Item, 0)
|
||||
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
|
||||
}
|
||||
|
||||
for _, blob := range resp.Segment.BlobItems {
|
||||
items = append(items, Item{
|
||||
Key: *blob.Name,
|
||||
ModifiedTime: *blob.Properties.LastModified,
|
||||
Size: *blob.Properties.ContentLength,
|
||||
IsDirectory: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Remove deletes a blob from Azure Blob Storage.
|
||||
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
|
||||
_, err := a.client.DeleteBlob(ctx, a.container, normalizePathSeparators(remotePath), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure: failed to remove blob from container %s: %w", a.container, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases resources.
|
||||
func (a *AzureStorage) Close() error {
|
||||
return nil
|
||||
}
|
||||
251
ftp.go
Normal file
251
ftp.go
Normal file
@@ -0,0 +1,251 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
)
|
||||
|
||||
// FTPStorage implements Storage interface for FTP.
|
||||
type FTPStorage struct {
|
||||
host string
|
||||
port string
|
||||
baseDir string
|
||||
client *ftp.ServerConn
|
||||
}
|
||||
|
||||
// NewFTPStorage creates a new FTP storage instance.
|
||||
func NewFTPStorage(config Config) (*FTPStorage, error) {
|
||||
if err := validateConfig(config, map[string]string{
|
||||
"host": config.FTPHost,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("ftp: %w", err)
|
||||
}
|
||||
|
||||
port := config.FTPPort
|
||||
if port == "" {
|
||||
port = "21"
|
||||
}
|
||||
|
||||
// Connect to FTP server
|
||||
hostPort := fmt.Sprintf("%s:%s", config.FTPHost, port)
|
||||
conn, err := ftp.Dial(hostPort, ftp.DialWithTimeout(30*time.Second))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
|
||||
}
|
||||
|
||||
// Login
|
||||
if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
|
||||
conn.Quit()
|
||||
return nil, fmt.Errorf("ftp: login failed: %w", err)
|
||||
}
|
||||
|
||||
return &FTPStorage{
|
||||
host: config.FTPHost,
|
||||
port: port,
|
||||
baseDir: config.FTPDirectory,
|
||||
client: conn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload uploads a file via FTP.
|
||||
func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
src, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ftp: failed to open local file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
// Create parent directory if needed
|
||||
remoteDir := filepath.Dir(fullRemotePath)
|
||||
remoteDir = normalizePathSeparators(remoteDir)
|
||||
|
||||
if remoteDir != "." && remoteDir != "/" {
|
||||
f.ensureDirectory(remoteDir)
|
||||
|
||||
}
|
||||
|
||||
// Upload file
|
||||
if err := f.client.Stor(fullRemotePath, src); err != nil {
|
||||
return fmt.Errorf("ftp: failed to upload file %s: %w", fullRemotePath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file via FTP.
|
||||
func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
// Get the file from FTP server
|
||||
resp, err := f.client.Retr(fullRemotePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ftp: failed to retrieve file %s: %w", fullRemotePath, err)
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
// Create local file
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ftp: failed to create local file: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(dst, resp); err != nil {
|
||||
return fmt.Errorf("ftp: failed to transfer data: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists files via FTP with the given prefix.
|
||||
func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
items := make([]Item, 0)
|
||||
|
||||
// Normalize the base directory
|
||||
baseDir := normalizePathSeparators(f.baseDir)
|
||||
if baseDir == "" {
|
||||
baseDir = "/"
|
||||
}
|
||||
|
||||
// Walk the directory
|
||||
if err := f.walkDirectory(baseDir, prefix, &items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// walkDirectory recursively walks FTP directory structure.
|
||||
func (f *FTPStorage) walkDirectory(dir string, prefix string, items *[]Item) error {
|
||||
entries, err := f.client.List(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
// Get relative path
|
||||
var relPath string
|
||||
if f.baseDir == "" || f.baseDir == "/" {
|
||||
relPath = filepath.Join(strings.TrimPrefix(dir, "/"), entry.Name)
|
||||
} else {
|
||||
fullPath := filepath.Join(dir, entry.Name)
|
||||
relPath = strings.TrimPrefix(fullPath, f.baseDir)
|
||||
relPath = strings.TrimPrefix(relPath, "/")
|
||||
}
|
||||
|
||||
// Normalize separators for comparison
|
||||
relPath = normalizePathSeparators(relPath)
|
||||
|
||||
// Skip if doesn't match prefix
|
||||
if prefix != "" && !strings.HasPrefix(relPath, prefix) {
|
||||
// For directories, we might need to walk them to find matching files
|
||||
if entry.Type == ftp.EntryTypeFolder {
|
||||
fullPath := filepath.Join(dir, entry.Name)
|
||||
fullPath = normalizePathSeparators(fullPath)
|
||||
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
|
||||
// Log but don't fail on subdirectory errors
|
||||
continue
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
item := Item{
|
||||
Key: relPath,
|
||||
ModifiedTime: entry.Time,
|
||||
Size: int64(entry.Size),
|
||||
IsDirectory: entry.Type == ftp.EntryTypeFolder,
|
||||
}
|
||||
*items = append(*items, item)
|
||||
|
||||
// Recursively walk directories
|
||||
if entry.Type == ftp.EntryTypeFolder {
|
||||
fullPath := filepath.Join(dir, entry.Name)
|
||||
fullPath = normalizePathSeparators(fullPath)
|
||||
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
|
||||
// Log but don't fail on subdirectory errors
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove deletes a file via FTP.
|
||||
func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
if err := f.client.Delete(fullRemotePath); err != nil {
|
||||
return fmt.Errorf("ftp: failed to remove %s: %w", fullRemotePath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases resources.
|
||||
func (f *FTPStorage) Close() error {
|
||||
if err := f.client.Quit(); err != nil {
|
||||
return fmt.Errorf("ftp: failed to close connection: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureDirectory creates a directory path if it doesn't exist.
|
||||
func (f *FTPStorage) ensureDirectory(path string) {
|
||||
parts := strings.Split(path, "/")
|
||||
current := ""
|
||||
|
||||
for _, part := range parts {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if current == "" {
|
||||
current = part
|
||||
} else {
|
||||
current = current + "/" + part
|
||||
}
|
||||
|
||||
// Try to create directory (ignore errors if it already exists)
|
||||
_ = f.client.MakeDir(current)
|
||||
}
|
||||
|
||||
}
|
||||
156
gcs.go
Normal file
156
gcs.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// GCSStorage implements Storage interface for Google Cloud Storage.
|
||||
type GCSStorage struct {
|
||||
client *storage.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewGCSStorage creates a new Google Cloud Storage instance.
|
||||
func NewGCSStorage(config Config) (*GCSStorage, error) {
|
||||
if err := validateConfig(config, map[string]string{
|
||||
"bucket": config.GCSBucket,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("gcs: %w", err)
|
||||
}
|
||||
|
||||
options := make([]option.ClientOption, 0)
|
||||
|
||||
if config.GCSEndpoint != "" {
|
||||
options = append(options, option.WithEndpoint(config.GCSEndpoint))
|
||||
}
|
||||
|
||||
if config.GCSCredentialsFile != "" {
|
||||
options = append(options, option.WithCredentialsFile(config.GCSCredentialsFile))
|
||||
}
|
||||
|
||||
client, err := storage.NewClient(context.Background(), options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gcs: failed to create client: %w", err)
|
||||
}
|
||||
|
||||
return &GCSStorage{
|
||||
client: client,
|
||||
bucket: config.GCSBucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload uploads a file to GCS.
|
||||
func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gcs: failed to open local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("gcs: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewWriter(ctx)
|
||||
defer func(obj *storage.Writer) {
|
||||
err = obj.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("gcs: failed to close object writer: %v\n", err)
|
||||
}
|
||||
}(obj)
|
||||
|
||||
if _, err := io.Copy(obj, file); err != nil {
|
||||
return fmt.Errorf("gcs: failed to write data: %w", err)
|
||||
}
|
||||
|
||||
if err = obj.Close(); err != nil {
|
||||
return fmt.Errorf("gcs: failed to finalize upload: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file from GCS.
|
||||
func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
file, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gcs: failed to create local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("gcs: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
obj, err := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewReader(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gcs: failed to create reader: %w", err)
|
||||
}
|
||||
defer func(obj *storage.Reader) {
|
||||
err = obj.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("gcs: failed to close object reader: %v\n", err)
|
||||
}
|
||||
}(obj)
|
||||
|
||||
if _, err = io.Copy(file, obj); err != nil {
|
||||
return fmt.Errorf("gcs: failed to read data: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists objects in GCS with the given prefix.
|
||||
func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
items := make([]Item, 0)
|
||||
|
||||
query := &storage.Query{
|
||||
Prefix: normalizePathSeparators(prefix),
|
||||
}
|
||||
|
||||
it := g.client.Bucket(g.bucket).Objects(ctx, query)
|
||||
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gcs: failed to iterate objects: %w", err)
|
||||
}
|
||||
|
||||
items = append(items, Item{
|
||||
Key: attrs.Name,
|
||||
ModifiedTime: attrs.Updated,
|
||||
Size: attrs.Size,
|
||||
IsDirectory: false,
|
||||
})
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Remove deletes an object from GCS.
|
||||
func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
|
||||
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath))
|
||||
|
||||
if err := obj.Delete(ctx); err != nil {
|
||||
return fmt.Errorf("gcs: failed to remove object from bucket %s: %w", g.bucket, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases resources.
|
||||
func (g *GCSStorage) Close() error {
|
||||
return g.client.Close()
|
||||
}
|
||||
67
go.mod
67
go.mod
@@ -1,22 +1,67 @@
|
||||
module github.com/jkaninda/go-storage
|
||||
|
||||
go 1.21.0
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/spf13/pflag v1.0.5
|
||||
cloud.google.com/go/storage v1.60.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||
github.com/aws/aws-sdk-go v1.55.7
|
||||
github.com/bramvdbogaerde/go-scp v1.5.0
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
golang.org/x/crypto v0.47.0
|
||||
google.golang.org/api v0.267.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go v1.55.3 // indirect
|
||||
github.com/bramvdbogaerde/go-scp v1.5.0 // indirect
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hpcloud/tail v1.0.0 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
golang.org/x/crypto v0.18.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
gopkg.in/fsnotify.v1 v1.4.7 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
|
||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.78.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
198
go.sum
198
go.sum
@@ -1,68 +1,170 @@
|
||||
github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
|
||||
github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
|
||||
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
|
||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
||||
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
||||
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
||||
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
||||
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
||||
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
|
||||
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
|
||||
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
||||
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
||||
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
|
||||
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
||||
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0=
|
||||
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
|
||||
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
|
||||
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
|
||||
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
|
||||
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
|
||||
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
|
||||
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
|
||||
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
|
||||
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
|
||||
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
||||
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
|
||||
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
|
||||
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
348
local.go
Normal file
348
local.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// LocalStorage implements the Storage interface using the local filesystem.
|
||||
// It is safe for concurrent use as it relies on filesystem operations which
|
||||
// are handled by the OS.
|
||||
type LocalStorage struct {
|
||||
basePath string
|
||||
}
|
||||
|
||||
// NewLocalStorage creates a new LocalStorage instance with the given base directory.
|
||||
// The base directory will be created if it doesn't exist.
|
||||
func NewLocalStorage(basePath string) (*LocalStorage, error) {
|
||||
absPath, err := filepath.Abs(basePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve absolute path: %w", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(absPath, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create base directory: %w", err)
|
||||
}
|
||||
|
||||
return &LocalStorage{
|
||||
basePath: absPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolvePath safely resolves a remote path to an absolute local path,
|
||||
// ensuring it stays within the base directory.
|
||||
func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
|
||||
// Clean the path to remove any .. or . components
|
||||
cleanPath := filepath.Clean(remotePath)
|
||||
|
||||
// Remove leading slashes to make it relative
|
||||
cleanPath = strings.TrimPrefix(cleanPath, string(filepath.Separator))
|
||||
|
||||
// Join with base path
|
||||
fullPath := filepath.Join(s.basePath, cleanPath)
|
||||
|
||||
// Verify the resolved path is still within basePath
|
||||
if !strings.HasPrefix(fullPath, s.basePath) {
|
||||
return "", fmt.Errorf("path traversal attempt detected: %s", remotePath)
|
||||
}
|
||||
|
||||
return fullPath, nil
|
||||
}
|
||||
|
||||
// Upload uploads a local file to the storage with the given target name.
|
||||
func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
targetPath, err := s.resolvePath(remotePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open source file
|
||||
srcFile, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file: %w", err)
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
// Create parent directories if needed
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create parent directories: %w", err)
|
||||
}
|
||||
|
||||
// Create destination file
|
||||
dstFile, err := os.Create(targetPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create destination file: %w", err)
|
||||
}
|
||||
defer dstFile.Close()
|
||||
|
||||
// Copy with context cancellation checks
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Clean up partial file on cancellation
|
||||
os.Remove(targetPath)
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
n, readErr := srcFile.Read(buf)
|
||||
if n > 0 {
|
||||
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
|
||||
os.Remove(targetPath)
|
||||
return fmt.Errorf("failed to write to destination: %w", writeErr)
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
os.Remove(targetPath)
|
||||
return fmt.Errorf("failed to read source file: %w", readErr)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file from storage to a local path.
|
||||
func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
sourcePath, err := s.resolvePath(remotePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open source file
|
||||
srcFile, err := os.Open(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open remote file: %w", err)
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
// Create parent directories for local path if needed
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create local parent directories: %w", err)
|
||||
}
|
||||
|
||||
// Create destination file
|
||||
dstFile, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create local file: %w", err)
|
||||
}
|
||||
defer dstFile.Close()
|
||||
|
||||
// Copy with context cancellation checks
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Clean up partial file on cancellation
|
||||
os.Remove(localPath)
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
n, readErr := srcFile.Read(buf)
|
||||
if n > 0 {
|
||||
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
|
||||
os.Remove(localPath)
|
||||
return fmt.Errorf("failed to write to local file: %w", writeErr)
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
os.Remove(localPath)
|
||||
return fmt.Errorf("failed to read remote file: %w", readErr)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns all items in the storage that match the given prefix.
|
||||
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var items []Item
|
||||
|
||||
// Clean and resolve the prefix path
|
||||
searchPath := s.basePath
|
||||
cleanPrefix := strings.TrimPrefix(filepath.Clean(prefix), string(filepath.Separator))
|
||||
|
||||
if cleanPrefix != "" && cleanPrefix != "." {
|
||||
searchPath = filepath.Join(s.basePath, cleanPrefix)
|
||||
}
|
||||
|
||||
// Check if the search path exists
|
||||
info, err := os.Stat(searchPath)
|
||||
if os.IsNotExist(err) {
|
||||
// If the exact path doesn't exist, try to find files with this prefix
|
||||
// by walking the parent directory
|
||||
parentDir := filepath.Dir(searchPath)
|
||||
baseName := filepath.Base(searchPath)
|
||||
|
||||
if _, err := os.Stat(parentDir); os.IsNotExist(err) {
|
||||
return items, nil // Return empty list if parent doesn't exist
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(parentDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if strings.HasPrefix(entry.Name(), baseName) {
|
||||
fullPath := filepath.Join(parentDir, entry.Name())
|
||||
item, err := s.fileInfoToItem(fullPath)
|
||||
if err != nil {
|
||||
continue // Skip files we can't stat
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat path: %w", err)
|
||||
}
|
||||
|
||||
// If it's a file, return just that file
|
||||
if !info.IsDir() {
|
||||
item, err := s.fileInfoToItem(searchPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []Item{item}, nil
|
||||
}
|
||||
|
||||
// Walk the directory
|
||||
err = filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil // Skip entries with errors
|
||||
}
|
||||
|
||||
item, err := s.fileInfoToItem(path)
|
||||
if err != nil {
|
||||
return nil // Skip files we can't stat
|
||||
}
|
||||
items = append(items, item)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to walk directory: %w", err)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// fileInfoToItem converts a file path to an Item.
|
||||
func (s *LocalStorage) fileInfoToItem(path string) (Item, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return Item{}, err
|
||||
}
|
||||
|
||||
// Get relative path from base
|
||||
relPath, err := filepath.Rel(s.basePath, path)
|
||||
if err != nil {
|
||||
relPath = path
|
||||
}
|
||||
|
||||
// Normalize to forward slashes for consistency
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
size := info.Size()
|
||||
if info.IsDir() {
|
||||
size = 0
|
||||
}
|
||||
|
||||
return Item{
|
||||
Key: relPath,
|
||||
ModifiedTime: info.ModTime(),
|
||||
Size: size,
|
||||
IsDirectory: info.IsDir(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Remove deletes a file from storage.
|
||||
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
targetPath, err := s.resolvePath(remotePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if file exists
|
||||
info, err := os.Stat(targetPath)
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("file not found: %s", remotePath)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
|
||||
// Remove file or directory
|
||||
if info.IsDir() {
|
||||
if err := os.RemoveAll(targetPath); err != nil {
|
||||
return fmt.Errorf("failed to remove directory: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := os.Remove(targetPath); err != nil {
|
||||
return fmt.Errorf("failed to remove file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases any resources held by the storage implementation.
|
||||
// For LocalStorage, this is a no-op as there are no persistent resources.
|
||||
func (s *LocalStorage) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// BasePath returns the base directory path of the storage.
|
||||
func (s *LocalStorage) BasePath() string {
|
||||
return s.basePath
|
||||
}
|
||||
161
pkg/azure/azure.go
Normal file
161
pkg/azure/azure.go
Normal file
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/jkaninda/go-storage/pkg"
|
||||
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type azureStorage struct {
|
||||
*pkg.Backend
|
||||
client *azblob.Client
|
||||
containerName string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
AccountName string
|
||||
AccountKey string
|
||||
ContainerName string
|
||||
LocalPath string
|
||||
RemotePath string
|
||||
}
|
||||
|
||||
// createClient creates FTP Client
|
||||
func createClient(conf Config) (*azblob.Client, error) {
|
||||
// Create the service URL
|
||||
credential, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create service URL")
|
||||
}
|
||||
serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName)
|
||||
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create client")
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// NewStorage creates new Storage
|
||||
func NewStorage(conf Config) (pkg.Storage, error) {
|
||||
client, err := createClient(conf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &azureStorage{
|
||||
client: client,
|
||||
containerName: conf.ContainerName,
|
||||
Backend: &pkg.Backend{
|
||||
RemotePath: conf.RemotePath,
|
||||
LocalPath: conf.LocalPath,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Copy copies file to Azure Blob Storage
|
||||
func (s azureStorage) Copy(fileName string) error {
|
||||
filePath := filepath.Join(s.LocalPath, fileName)
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
||||
}
|
||||
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
blobName := filepath.Join(s.RemotePath, fileName)
|
||||
|
||||
// Get the container client
|
||||
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
|
||||
|
||||
// Get the blob client
|
||||
blobClient := containerClient.NewBlockBlobClient(blobName)
|
||||
|
||||
// Upload the file
|
||||
_, err = blobClient.UploadFile(context.Background(), file, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to upload file: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CopyFrom copies a file from Azure Blob Storage to local storage
|
||||
func (s azureStorage) CopyFrom(blobName string) error {
|
||||
|
||||
// Get the container client
|
||||
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
|
||||
|
||||
// Get the blob client
|
||||
blobClient := containerClient.NewBlockBlobClient(filepath.Join(s.RemotePath, blobName))
|
||||
|
||||
// Download the blob
|
||||
downloadResponse, err := blobClient.DownloadStream(context.Background(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the file to save the blob data
|
||||
downloadFile, err := os.Create(filepath.Join(s.LocalPath, blobName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func(downloadFile *os.File) {
|
||||
err := downloadFile.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to close file: %v", err)
|
||||
}
|
||||
}(downloadFile)
|
||||
|
||||
// Write the blob data to the file
|
||||
_, err = io.Copy(downloadFile, downloadResponse.Body)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to write blob to file: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prune deletes old backup created more than specified days
|
||||
func (s azureStorage) Prune(retentionDays int) error {
|
||||
fmt.Println("Deleting old backup from a remote server is not implemented yet")
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// Name returns the storage name
|
||||
func (s azureStorage) Name() string {
|
||||
return "azure"
|
||||
}
|
||||
@@ -1,3 +1,27 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
@@ -20,14 +44,14 @@ type Config struct {
|
||||
Host string
|
||||
User string
|
||||
Password string
|
||||
Port string
|
||||
Port int
|
||||
LocalPath string
|
||||
RemotePath string
|
||||
}
|
||||
|
||||
// createClient creates FTP Client
|
||||
func createClient(conf Config) (*ftp.ServerConn, error) {
|
||||
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
|
||||
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%d", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
|
||||
}
|
||||
@@ -58,14 +82,24 @@ func NewStorage(conf Config) (pkg.Storage, error) {
|
||||
// Copy copies file to the remote server
|
||||
func (s ftpStorage) Copy(fileName string) error {
|
||||
ftpClient := s.client
|
||||
defer ftpClient.Quit()
|
||||
defer func(ftpClient *ftp.ServerConn) {
|
||||
err := ftpClient.Quit()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(ftpClient)
|
||||
|
||||
filePath := filepath.Join(s.LocalPath, fileName)
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
||||
}
|
||||
defer file.Close()
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||
err = ftpClient.Stor(remoteFilePath, file)
|
||||
@@ -80,21 +114,36 @@ func (s ftpStorage) Copy(fileName string) error {
|
||||
func (s ftpStorage) CopyFrom(fileName string) error {
|
||||
ftpClient := s.client
|
||||
|
||||
defer ftpClient.Quit()
|
||||
defer func(ftpClient *ftp.ServerConn) {
|
||||
err := ftpClient.Quit()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(ftpClient)
|
||||
|
||||
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||
r, err := ftpClient.Retr(remoteFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
|
||||
}
|
||||
defer r.Close()
|
||||
defer func(r *ftp.Response) {
|
||||
err := r.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(r)
|
||||
|
||||
localFilePath := filepath.Join(s.LocalPath, fileName)
|
||||
outFile, err := os.Create(localFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
defer func(outFile *os.File) {
|
||||
err := outFile.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(outFile)
|
||||
|
||||
_, err = io.Copy(outFile, r)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,3 +1,27 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
@@ -92,7 +116,12 @@ func copyFile(src, dst string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
defer func(in *os.File) {
|
||||
err := in.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(in)
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
@@ -101,7 +130,10 @@ func copyFile(src, dst string) error {
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
out.Close()
|
||||
err := out.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
|
||||
@@ -45,7 +45,12 @@ func createFile(fileName, content string) ([]byte, error) {
|
||||
fmt.Println("Error creating file:", err)
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
// Write the message to the file
|
||||
_, err = file.WriteString(content)
|
||||
|
||||
50
pkg/s3/s3.go
50
pkg/s3/s3.go
@@ -1,3 +1,27 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
@@ -70,7 +94,12 @@ func (s s3Storage) Copy(fileName string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
fileInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
@@ -78,7 +107,10 @@ func (s s3Storage) Copy(fileName string) error {
|
||||
}
|
||||
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||
buffer := make([]byte, fileInfo.Size())
|
||||
file.Read(buffer)
|
||||
_, err = file.Read(buffer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes := bytes.NewReader(buffer)
|
||||
fileType := http.DetectContentType(buffer)
|
||||
|
||||
@@ -102,7 +134,13 @@ func (s s3Storage) CopyFrom(fileName string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("Error closing file: %v\n", err)
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||
|
||||
@@ -140,16 +178,16 @@ func (s s3Storage) Prune(retentionDays int) error {
|
||||
Key: object.Key,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("failed to delete object %s: %v", *object.Key, err)
|
||||
fmt.Printf("failed to delete object %s: %v\n", *object.Key, err)
|
||||
} else {
|
||||
fmt.Printf("Deleted object %s", *object.Key)
|
||||
fmt.Printf("Deleted object %s\n", *object.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return !lastPage
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list objects: %v", err)
|
||||
return fmt.Errorf("failed to list objects: %v\n", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,3 +1,27 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package ssh
|
||||
|
||||
import (
|
||||
@@ -22,7 +46,7 @@ type Config struct {
|
||||
Host string
|
||||
User string
|
||||
Password string
|
||||
Port string
|
||||
Port int
|
||||
IdentifyFile string
|
||||
LocalPath string
|
||||
RemotePath string
|
||||
@@ -30,15 +54,15 @@ type Config struct {
|
||||
|
||||
// createClient creates SSH Client
|
||||
func createClient(conf Config) (scp.Client, error) {
|
||||
if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) {
|
||||
if _, err := os.Stat(conf.IdentifyFile); !os.IsNotExist(err) {
|
||||
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
|
||||
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
||||
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
|
||||
} else {
|
||||
if conf.Password == "" {
|
||||
return scp.Client{}, errors.New("ssh password required")
|
||||
}
|
||||
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
|
||||
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
||||
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
|
||||
|
||||
}
|
||||
}
|
||||
@@ -97,7 +121,12 @@ func (s sshStorage) CopyFrom(fileName string) error {
|
||||
if err != nil {
|
||||
return errors.New("couldn't open the output file")
|
||||
}
|
||||
defer file.Close()
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(file)
|
||||
|
||||
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
|
||||
|
||||
|
||||
1
pkg/ssh/ssh_test.go
Normal file
1
pkg/ssh/ssh_test.go
Normal file
@@ -0,0 +1 @@
|
||||
package ssh
|
||||
@@ -1,3 +1,27 @@
|
||||
/*
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Jonas Kaninda
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
package pkg
|
||||
|
||||
type Storage interface {
|
||||
@@ -7,8 +31,8 @@ type Storage interface {
|
||||
Name() string
|
||||
}
|
||||
type Backend struct {
|
||||
//Local Path
|
||||
// Local Path
|
||||
LocalPath string
|
||||
//Remote path or Destination path
|
||||
// Remote path or Destination path
|
||||
RemotePath string
|
||||
}
|
||||
|
||||
187
s3.go
Normal file
187
s3.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// S3Storage implements Storage interface for Amazon S3.
|
||||
type S3Storage struct {
|
||||
session *session.Session
|
||||
bucket string
|
||||
region string
|
||||
}
|
||||
|
||||
// NewS3Storage creates a new S3 storage instance.
|
||||
func NewS3Storage(config Config) (*S3Storage, error) {
|
||||
if err := validateConfig(config, map[string]string{
|
||||
"bucket": config.S3Bucket,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("s3: %w", err)
|
||||
}
|
||||
|
||||
awsConfig := aws.NewConfig()
|
||||
|
||||
if config.S3Region != "" {
|
||||
awsConfig = awsConfig.WithRegion(config.S3Region)
|
||||
}
|
||||
|
||||
if config.S3KeyID != "" && config.S3Secret != "" {
|
||||
awsConfig = awsConfig.WithCredentials(
|
||||
credentials.NewStaticCredentials(config.S3KeyID, config.S3Secret, ""),
|
||||
)
|
||||
}
|
||||
|
||||
if config.S3Endpoint != "" {
|
||||
awsConfig = awsConfig.WithEndpoint(config.S3Endpoint)
|
||||
}
|
||||
|
||||
if config.S3ForcePath {
|
||||
awsConfig = awsConfig.WithS3ForcePathStyle(true)
|
||||
}
|
||||
|
||||
if config.S3DisableTLS {
|
||||
awsConfig = awsConfig.WithDisableSSL(true)
|
||||
}
|
||||
|
||||
sessionOpts := session.Options{
|
||||
Config: *awsConfig,
|
||||
SharedConfigState: session.SharedConfigEnable,
|
||||
}
|
||||
|
||||
if config.S3Profile != "" {
|
||||
sessionOpts.Profile = config.S3Profile
|
||||
}
|
||||
|
||||
sess, err := session.NewSessionWithOptions(sessionOpts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: failed to create session: %w", err)
|
||||
}
|
||||
|
||||
return &S3Storage{
|
||||
session: sess,
|
||||
bucket: config.S3Bucket,
|
||||
region: config.S3Region,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload uploads a file to S3.
|
||||
func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3: failed to open local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("s3: warning: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
uploader := s3manager.NewUploader(s.session)
|
||||
|
||||
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||
Body: file,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3: failed to upload to bucket %s: %w", s.bucket, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file from S3.
|
||||
func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
file, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3: failed to create local file: %w", err)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("s3: warning: failed to close file: %v\n", err)
|
||||
}
|
||||
}(file)
|
||||
|
||||
downloader := s3manager.NewDownloader(s.session)
|
||||
|
||||
_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3: failed to download from bucket %s: %w", s.bucket, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists objects in S3 with the given prefix.
|
||||
func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
svc := s3.New(s.session)
|
||||
items := make([]Item, 0)
|
||||
|
||||
var continuationToken *string
|
||||
|
||||
for {
|
||||
input := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(normalizePathSeparators(prefix)),
|
||||
ContinuationToken: continuationToken,
|
||||
}
|
||||
|
||||
resp, err := svc.ListObjectsV2WithContext(ctx, input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
|
||||
}
|
||||
|
||||
for _, obj := range resp.Contents {
|
||||
items = append(items, Item{
|
||||
Key: *obj.Key,
|
||||
ModifiedTime: *obj.LastModified,
|
||||
Size: *obj.Size,
|
||||
IsDirectory: false,
|
||||
})
|
||||
}
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
|
||||
continuationToken = resp.NextContinuationToken
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Remove deletes an object from S3.
|
||||
func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
|
||||
svc := s3.New(s.session)
|
||||
|
||||
_, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3: failed to remove object from bucket %s: %w", s.bucket, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases resources.
|
||||
func (s *S3Storage) Close() error {
|
||||
return nil
|
||||
}
|
||||
350
sftp.go
Normal file
350
sftp.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/agent"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
// SFTPStorage implements Storage interface for SFTP.
|
||||
type SFTPStorage struct {
|
||||
host string
|
||||
baseDir string
|
||||
sshClient *ssh.Client
|
||||
client *sftp.Client
|
||||
}
|
||||
|
||||
// NewSFTPStorage creates a new SFTP storage instance.
|
||||
func NewSFTPStorage(config Config) (*SFTPStorage, error) {
|
||||
if err := validateConfig(config, map[string]string{
|
||||
"host": config.SFTPHost,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("sftp: %w", err)
|
||||
}
|
||||
|
||||
port := config.SFTPPort
|
||||
if port == "" {
|
||||
port = "22"
|
||||
}
|
||||
|
||||
username := config.SFTPUsername
|
||||
if username == "" {
|
||||
currentUser, err := user.Current()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sftp: failed to get current user: %w", err)
|
||||
}
|
||||
username = currentUser.Username
|
||||
}
|
||||
|
||||
password := config.SFTPPassword
|
||||
if password == "" {
|
||||
password = os.Getenv("PGBK_SSH_PASS")
|
||||
}
|
||||
|
||||
// Build authentication methods
|
||||
authMethods := make([]ssh.AuthMethod, 0)
|
||||
|
||||
// Password authentication (if no identity file is specified)
|
||||
if config.SFTPIdentityFile == "" && password != "" {
|
||||
authMethods = append(authMethods, ssh.Password(password))
|
||||
}
|
||||
|
||||
// Public key authentication
|
||||
signers, err := getSSHSigners(config.SFTPIdentityFile, password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sftp: %w", err)
|
||||
}
|
||||
if len(signers) > 0 {
|
||||
authMethods = append(authMethods, ssh.PublicKeys(signers...))
|
||||
}
|
||||
|
||||
// Build SSH client config
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: username,
|
||||
Auth: authMethods,
|
||||
HostKeyCallback: getHostKeyCallback(config.SFTPIgnoreKnownHosts),
|
||||
}
|
||||
|
||||
// Connect to SSH server
|
||||
hostPort := fmt.Sprintf("%s:%s", config.SFTPHost, port)
|
||||
sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
|
||||
}
|
||||
|
||||
// Open SFTP session
|
||||
sftpClient, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
sshClient.Close()
|
||||
return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
|
||||
}
|
||||
|
||||
baseDir := config.SFTPDirectory
|
||||
if baseDir == "" {
|
||||
wd, err := sftpClient.Getwd()
|
||||
if err != nil {
|
||||
sftpClient.Close()
|
||||
sshClient.Close()
|
||||
return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
|
||||
}
|
||||
baseDir = wd
|
||||
}
|
||||
|
||||
return &SFTPStorage{
|
||||
host: config.SFTPHost,
|
||||
baseDir: baseDir,
|
||||
sshClient: sshClient,
|
||||
client: sftpClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload uploads a file via SFTP.
|
||||
func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||
src, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: failed to open local file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
// Create parent directory if needed
|
||||
remoteDir := filepath.Dir(fullRemotePath)
|
||||
remoteDir = normalizePathSeparators(remoteDir)
|
||||
|
||||
if remoteDir != "." && remoteDir != "/" {
|
||||
if err := s.client.MkdirAll(remoteDir); err != nil {
|
||||
return fmt.Errorf("sftp: failed to create directory %s: %w", remoteDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
dst, err := s.client.Create(fullRemotePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: failed to create remote file %s: %w", fullRemotePath, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return fmt.Errorf("sftp: failed to transfer data: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download downloads a file via SFTP.
|
||||
func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: failed to create local file: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
src, err := s.client.Open(fullRemotePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: failed to open remote file %s: %w", fullRemotePath, err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return fmt.Errorf("sftp: failed to transfer data: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists files via SFTP with the given prefix.
|
||||
func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||
items := make([]Item, 0)
|
||||
|
||||
baseDir := normalizePathSeparators(s.baseDir)
|
||||
|
||||
walker := s.client.Walk(baseDir)
|
||||
for walker.Step() {
|
||||
if err := walker.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sftp: walk error: %w", err)
|
||||
}
|
||||
|
||||
// Get relative path
|
||||
path := walker.Path()
|
||||
relPath, err := filepath.Rel(baseDir, path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Normalize separators for comparison
|
||||
relPath = normalizePathSeparators(relPath)
|
||||
|
||||
if !strings.HasPrefix(relPath, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
stat := walker.Stat()
|
||||
items = append(items, Item{
|
||||
Key: relPath,
|
||||
ModifiedTime: stat.ModTime(),
|
||||
Size: stat.Size(),
|
||||
IsDirectory: stat.IsDir(),
|
||||
})
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Remove deletes a file via SFTP.
|
||||
func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
|
||||
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||
|
||||
if err := s.client.Remove(fullRemotePath); err != nil {
|
||||
return fmt.Errorf("sftp: failed to remove %s: %w", fullRemotePath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases resources.
|
||||
func (s *SFTPStorage) Close() error {
|
||||
if err := s.client.Close(); err != nil {
|
||||
s.sshClient.Close()
|
||||
return err
|
||||
}
|
||||
return s.sshClient.Close()
|
||||
}
|
||||
|
||||
// getSSHSigners returns SSH signers from identity file and SSH agent.
|
||||
func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
|
||||
signers := make([]ssh.Signer, 0)
|
||||
|
||||
// Load identity file if provided
|
||||
if identityFile != "" {
|
||||
path, err := expandHomeDir(identityFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to expand identity file path: %w", err)
|
||||
}
|
||||
|
||||
keyData, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read identity file %s: %w", path, err)
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(keyData)
|
||||
if err != nil {
|
||||
var passErr *ssh.PassphraseMissingError
|
||||
if errors.As(err, &passErr) {
|
||||
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to parse identity file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
signers = append(signers, signer)
|
||||
}
|
||||
|
||||
// Try to get keys from SSH agent
|
||||
socket := os.Getenv("SSH_AUTH_SOCK")
|
||||
if socket != "" {
|
||||
conn, err := net.Dial("unix", socket)
|
||||
if err == nil {
|
||||
agentClient := agent.NewClient(conn)
|
||||
agentSigners, err := agentClient.Signers()
|
||||
if err == nil {
|
||||
signers = append(signers, agentSigners...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return signers, nil
|
||||
}
|
||||
|
||||
// getHostKeyCallback returns appropriate host key callback.
|
||||
func getHostKeyCallback(ignoreHostKey bool) ssh.HostKeyCallback {
|
||||
if ignoreHostKey {
|
||||
return ssh.InsecureIgnoreHostKey()
|
||||
}
|
||||
|
||||
knownHostsFiles := make([]string, 0)
|
||||
for _, p := range []string{"/etc/ssh/ssh_known_hosts", "~/.ssh/known_hosts"} {
|
||||
path, err := expandHomeDir(p)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
knownHostsFiles = append(knownHostsFiles, path)
|
||||
}
|
||||
}
|
||||
|
||||
if len(knownHostsFiles) == 0 {
|
||||
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||
return fmt.Errorf("no known_hosts files found for host key verification")
|
||||
}
|
||||
}
|
||||
|
||||
callback, err := knownhosts.New(knownHostsFiles...)
|
||||
if err != nil {
|
||||
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||
return fmt.Errorf("failed to load known_hosts: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return callback
|
||||
}
|
||||
|
||||
// expandHomeDir expands ~ in file paths to home directory.
|
||||
func expandHomeDir(path string) (string, error) {
|
||||
if !strings.HasPrefix(path, "~") {
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
|
||||
parts := strings.SplitN(path, "/", 2)
|
||||
username := strings.TrimPrefix(parts[0], "~")
|
||||
|
||||
var homeDir string
|
||||
var err error
|
||||
|
||||
if username == "" {
|
||||
homeDir, err = os.UserHomeDir()
|
||||
if err != nil || homeDir == "" {
|
||||
currentUser, err := user.Current()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get home directory: %w", err)
|
||||
}
|
||||
homeDir = currentUser.HomeDir
|
||||
}
|
||||
} else {
|
||||
userInfo, err := user.Lookup(username)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to lookup user %s: %w", username, err)
|
||||
}
|
||||
homeDir = userInfo.HomeDir
|
||||
}
|
||||
|
||||
if homeDir == "" {
|
||||
return "", fmt.Errorf("empty home directory")
|
||||
}
|
||||
|
||||
if len(parts) == 1 {
|
||||
return homeDir, nil
|
||||
}
|
||||
|
||||
return filepath.Join(homeDir, parts[1]), nil
|
||||
}
|
||||
140
storage.go
Normal file
140
storage.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Storage defines the interface for remote file storage operations.
|
||||
// All implementations must be safe for concurrent use.
|
||||
type Storage interface {
|
||||
// Upload uploads a local file to the remote storage with the given target name.
|
||||
Upload(ctx context.Context, localPath string, remotePath string) error
|
||||
|
||||
// Download downloads a file from remote storage to a local path.
|
||||
Download(ctx context.Context, remotePath string, localPath string) error
|
||||
|
||||
// List returns all items in the storage that match the given prefix.
|
||||
List(ctx context.Context, prefix string) ([]Item, error)
|
||||
|
||||
// Remove deletes a file from remote storage.
|
||||
Remove(ctx context.Context, remotePath string) error
|
||||
|
||||
// Close releases any resources held by the storage implementation.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Item represents a file or directory in remote storage.
|
||||
type Item struct {
|
||||
Key string // The path/key of the item
|
||||
ModifiedTime time.Time // Last modification time
|
||||
Size int64 // Size in bytes (0 for directories)
|
||||
IsDirectory bool // Whether this item is a directory
|
||||
}
|
||||
|
||||
// Config holds configuration options for creating storage instances.
|
||||
type Config struct {
|
||||
// S3 configuration
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Profile string
|
||||
S3KeyID string
|
||||
S3Secret string
|
||||
S3Endpoint string
|
||||
S3ForcePath bool
|
||||
S3DisableTLS bool
|
||||
|
||||
// B2 configuration
|
||||
B2KeyID string
|
||||
B2AppKey string
|
||||
B2Bucket string
|
||||
B2ConcurrentConnections int
|
||||
B2ForcePath bool
|
||||
|
||||
// Google Cloud Storage configuration
|
||||
GCSBucket string
|
||||
GCSEndpoint string
|
||||
GCSCredentialsFile string
|
||||
|
||||
// Azure configuration
|
||||
AzureAccount string
|
||||
AzureKey string
|
||||
AzureContainer string
|
||||
AzureEndpoint string
|
||||
|
||||
// SFTP configuration
|
||||
SFTPHost string
|
||||
SFTPPort string
|
||||
SFTPUsername string
|
||||
SFTPPassword string
|
||||
SFTPDirectory string
|
||||
SFTPIdentityFile string
|
||||
SFTPIgnoreKnownHosts bool
|
||||
|
||||
// FTP configuration
|
||||
FTPHost string
|
||||
FTPPort string
|
||||
FTPUsername string
|
||||
FTPPassword string
|
||||
FTPDirectory string
|
||||
}
|
||||
|
||||
// StorageType represents the type of storage backend.
|
||||
type StorageType string
|
||||
|
||||
const (
|
||||
StorageTypeS3 StorageType = "s3"
|
||||
StorageTypeGCS StorageType = "gcs"
|
||||
StorageTypeAzure StorageType = "azure"
|
||||
StorageTypeSFTP StorageType = "sftp"
|
||||
StorageTypeFTP StorageType = "ftp"
|
||||
StorageTypeLocal StorageType = "local"
|
||||
)
|
||||
|
||||
// NewStorage creates a new Storage instance based on the specified type and configuration.
|
||||
func NewStorage(storageType StorageType, config Config) (Storage, error) {
|
||||
switch storageType {
|
||||
case StorageTypeS3:
|
||||
return NewS3Storage(config)
|
||||
case StorageTypeGCS:
|
||||
return NewGCSStorage(config)
|
||||
case StorageTypeAzure:
|
||||
return NewAzureStorage(config)
|
||||
case StorageTypeSFTP:
|
||||
return NewSFTPStorage(config)
|
||||
case StorageTypeFTP:
|
||||
return NewFTPStorage(config)
|
||||
case StorageTypeLocal:
|
||||
return NewLocalStorage("/backups")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
|
||||
}
|
||||
}
|
||||
|
||||
// normalizePathSeparators converts OS-specific path separators to forward slashes.
|
||||
// This is necessary for cloud storage services which expect forward slashes.
|
||||
func normalizePathSeparators(path string) string {
|
||||
if os.PathSeparator == '/' {
|
||||
return path
|
||||
}
|
||||
return strings.ReplaceAll(path, string(os.PathSeparator), "/")
|
||||
}
|
||||
|
||||
// validateConfig checks if the required configuration fields are present.
|
||||
func validateConfig(_ Config, requiredFields map[string]string) error {
|
||||
var missing []string
|
||||
for field, value := range requiredFields {
|
||||
if value == "" {
|
||||
missing = append(missing, field)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
41
storage_test.go
Normal file
41
storage_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package go_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Test helpers
|
||||
func TestStorageInterface(t *testing.T) {
|
||||
var _ Storage = (*S3Storage)(nil)
|
||||
var _ Storage = (*GCSStorage)(nil)
|
||||
var _ Storage = (*AzureStorage)(nil)
|
||||
var _ Storage = (*SFTPStorage)(nil)
|
||||
}
|
||||
|
||||
func TestContextTimeout(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
config := Config{
|
||||
S3Region: "us-east-1",
|
||||
S3Bucket: "test-bucket",
|
||||
}
|
||||
|
||||
s3, err := NewStorage(StorageTypeS3, config)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create storage: %v", err)
|
||||
}
|
||||
defer func(s3 Storage) {
|
||||
err = s3.Close()
|
||||
if err != nil {
|
||||
t.Logf("Failed to close storage: %v", err)
|
||||
}
|
||||
}(s3)
|
||||
|
||||
err = s3.Upload(ctx, "test.txt", "remote-test.txt")
|
||||
if err != nil {
|
||||
t.Logf("Upload failed (expected in test): %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user