mirror of
https://github.com/jkaninda/go-storage.git
synced 2026-03-11 12:09:06 +01:00
Compare commits
24 Commits
v0.1.1
...
v0.2.0-rc1
| Author | SHA1 | Date | |
|---|---|---|---|
| fdbf0cd233 | |||
|
|
e5e68bdadf | ||
|
|
652d00ce3b | ||
|
|
c39d828fe6 | ||
|
|
b9d06c6529 | ||
|
|
373e8ada7a | ||
|
|
da0fc37905 | ||
|
|
cd01e0007c | ||
|
|
bcdcffed55 | ||
|
|
624c986a23 | ||
|
|
a06c6acdb8 | ||
|
|
538df008f7 | ||
|
|
10e9be1f49 | ||
|
|
367de149f2 | ||
| 1e36fb1884 | |||
|
|
da738264ab | ||
| 085bdf468f | |||
|
|
0a70da956f | ||
|
|
793b04340e | ||
|
|
1b147ebea5 | ||
|
|
2bcfd3aacf | ||
|
|
845e0cc43b | ||
|
|
d314c3854b | ||
|
|
2339db673d |
6
.github/dependabot.yml
vendored
Normal file
6
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: gomod
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
23
.github/workflows/lint.yml
vendored
Normal file
23
.github/workflows/lint.yml
vendored
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
name: Lint
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
name: Run on Ubuntu
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Clone the code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version: '~1.24'
|
||||||
|
|
||||||
|
- name: Run linter
|
||||||
|
uses: golangci/golangci-lint-action@v7
|
||||||
|
with:
|
||||||
|
version: v2.1.2
|
||||||
47
.golangci.yml
Normal file
47
.golangci.yml
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
version: "2"
|
||||||
|
run:
|
||||||
|
allow-parallel-runners: true
|
||||||
|
linters:
|
||||||
|
default: none
|
||||||
|
enable:
|
||||||
|
- copyloopvar
|
||||||
|
- dupl
|
||||||
|
- errcheck
|
||||||
|
- ginkgolinter
|
||||||
|
- goconst
|
||||||
|
- gocyclo
|
||||||
|
- govet
|
||||||
|
- ineffassign
|
||||||
|
#- lll
|
||||||
|
- misspell
|
||||||
|
- nakedret
|
||||||
|
- prealloc
|
||||||
|
- revive
|
||||||
|
- staticcheck
|
||||||
|
- unconvert
|
||||||
|
- unparam
|
||||||
|
- unused
|
||||||
|
settings:
|
||||||
|
revive:
|
||||||
|
rules:
|
||||||
|
- name: comment-spacings
|
||||||
|
exclusions:
|
||||||
|
generated: lax
|
||||||
|
rules:
|
||||||
|
- linters:
|
||||||
|
- dupl
|
||||||
|
- lll
|
||||||
|
path: /*
|
||||||
|
paths:
|
||||||
|
- third_party$
|
||||||
|
- builtin$
|
||||||
|
- examples$
|
||||||
|
formatters:
|
||||||
|
enable:
|
||||||
|
- gofmt
|
||||||
|
exclusions:
|
||||||
|
generated: lax
|
||||||
|
paths:
|
||||||
|
- third_party$
|
||||||
|
- builtin$
|
||||||
|
- examples$
|
||||||
30
README.md
30
README.md
@@ -6,6 +6,7 @@ Supported storage:
|
|||||||
- S3
|
- S3
|
||||||
- SSH
|
- SSH
|
||||||
- FTP
|
- FTP
|
||||||
|
- Azure Blob
|
||||||
|
|
||||||
```go
|
```go
|
||||||
go get github.com/jkaninda/go-storage
|
go get github.com/jkaninda/go-storage
|
||||||
@@ -55,7 +56,7 @@ if err != nil {
|
|||||||
```go
|
```go
|
||||||
sshStorage, err := ssh.NewStorage(ssh.Config{
|
sshStorage, err := ssh.NewStorage(ssh.Config{
|
||||||
Host: "",
|
Host: "",
|
||||||
Port: "",
|
Port: 22,
|
||||||
User: "",
|
User: "",
|
||||||
Password: "",
|
Password: "",
|
||||||
RemotePath: "",
|
RemotePath: "",
|
||||||
@@ -79,7 +80,7 @@ log.Fatalf("Error copying file, error %v", err)
|
|||||||
```go
|
```go
|
||||||
ftpStorage, err := ftp.NewStorage(ftp.Config{
|
ftpStorage, err := ftp.NewStorage(ftp.Config{
|
||||||
Host: "",
|
Host: "",
|
||||||
Port: "",
|
Port: 21,
|
||||||
User: "",
|
User: "",
|
||||||
Password: "",
|
Password: "",
|
||||||
RemotePath: "",
|
RemotePath: "",
|
||||||
@@ -98,3 +99,28 @@ if err != nil {
|
|||||||
log.Fatalf("Error copying file, error %v", err)
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Azure Blob storage
|
||||||
|
|
||||||
|
```go
|
||||||
|
azureStorage, err := azure.NewStorage(azure.Config{
|
||||||
|
ContainerName: '',
|
||||||
|
AccountName: '',
|
||||||
|
AccountKey: '',
|
||||||
|
RemotePath: '',
|
||||||
|
LocalPath: '',
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error creating Azure Blob storage storage: %s", err)
|
||||||
|
}
|
||||||
|
err = azureStorage.Copy(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error copying file: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download file from Azure Blob remote server
|
||||||
|
err = azureStorage.CopyFrom(finalFileName)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error copying file, error %v", err)
|
||||||
|
}
|
||||||
|
```
|
||||||
155
azure.go
Normal file
155
azure.go
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AzureStorage implements Storage interface for Azure Blob Storage.
|
||||||
|
type AzureStorage struct {
|
||||||
|
client *azblob.Client
|
||||||
|
container string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAzureStorage creates a new Azure Blob Storage instance.
|
||||||
|
func NewAzureStorage(config Config) (*AzureStorage, error) {
|
||||||
|
if err := validateConfig(config, map[string]string{
|
||||||
|
"container": config.AzureContainer,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("azure: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to get credentials from environment if not provided
|
||||||
|
account := config.AzureAccount
|
||||||
|
if account == "" {
|
||||||
|
account = os.Getenv("AZURE_STORAGE_ACCOUNT")
|
||||||
|
}
|
||||||
|
|
||||||
|
key := config.AzureKey
|
||||||
|
if key == "" {
|
||||||
|
key = os.Getenv("AZURE_STORAGE_KEY")
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := config.AzureEndpoint
|
||||||
|
if endpoint == "" {
|
||||||
|
endpoint = "blob.core.windows.net"
|
||||||
|
}
|
||||||
|
|
||||||
|
var client *azblob.Client
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if account == "" {
|
||||||
|
// Anonymous access
|
||||||
|
client, err = azblob.NewClientWithNoCredential(endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("azure: failed to create anonymous client: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Authenticated access
|
||||||
|
credential, err := azblob.NewSharedKeyCredential(account, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("azure: failed to create credentials: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
url := fmt.Sprintf("https://%s.%s", account, endpoint)
|
||||||
|
|
||||||
|
client, err = azblob.NewClientWithSharedKeyCredential(url, credential, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("azure: failed to create client: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &AzureStorage{
|
||||||
|
client: client,
|
||||||
|
container: config.AzureContainer,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a file to Azure Blob Storage.
|
||||||
|
func (a *AzureStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("azure: failed to open local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("azure: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
_, err = a.client.UploadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("azure: failed to upload to container %s: %w", a.container, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file from Azure Blob Storage.
|
||||||
|
func (a *AzureStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
file, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("azure: failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("azure: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
_, err = a.client.DownloadFile(ctx, a.container, normalizePathSeparators(remotePath), file, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("azure: failed to download from container %s: %w", a.container, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists blobs in Azure Blob Storage with the given prefix.
|
||||||
|
func (a *AzureStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
normalizedPrefix := normalizePathSeparators(prefix)
|
||||||
|
|
||||||
|
pager := a.client.NewListBlobsFlatPager(a.container, &azblob.ListBlobsFlatOptions{
|
||||||
|
Prefix: &normalizedPrefix,
|
||||||
|
})
|
||||||
|
|
||||||
|
items := make([]Item, 0)
|
||||||
|
|
||||||
|
for pager.More() {
|
||||||
|
resp, err := pager.NextPage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("azure: failed to list blobs in container %s: %w", a.container, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, blob := range resp.Segment.BlobItems {
|
||||||
|
items = append(items, Item{
|
||||||
|
Key: *blob.Name,
|
||||||
|
ModifiedTime: *blob.Properties.LastModified,
|
||||||
|
Size: *blob.Properties.ContentLength,
|
||||||
|
IsDirectory: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes a blob from Azure Blob Storage.
|
||||||
|
func (a *AzureStorage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
_, err := a.client.DeleteBlob(ctx, a.container, normalizePathSeparators(remotePath), nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("azure: failed to remove blob from container %s: %w", a.container, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources.
|
||||||
|
func (a *AzureStorage) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
251
ftp.go
Normal file
251
ftp.go
Normal file
@@ -0,0 +1,251 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jlaffaye/ftp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FTPStorage implements Storage interface for FTP.
|
||||||
|
type FTPStorage struct {
|
||||||
|
host string
|
||||||
|
port string
|
||||||
|
baseDir string
|
||||||
|
client *ftp.ServerConn
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFTPStorage creates a new FTP storage instance.
|
||||||
|
func NewFTPStorage(config Config) (*FTPStorage, error) {
|
||||||
|
if err := validateConfig(config, map[string]string{
|
||||||
|
"host": config.FTPHost,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("ftp: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
port := config.FTPPort
|
||||||
|
if port == "" {
|
||||||
|
port = "21"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect to FTP server
|
||||||
|
hostPort := fmt.Sprintf("%s:%s", config.FTPHost, port)
|
||||||
|
conn, err := ftp.Dial(hostPort, ftp.DialWithTimeout(30*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ftp: failed to connect to %s: %w", hostPort, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Login
|
||||||
|
if err := conn.Login(config.FTPUsername, config.FTPPassword); err != nil {
|
||||||
|
conn.Quit()
|
||||||
|
return nil, fmt.Errorf("ftp: login failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &FTPStorage{
|
||||||
|
host: config.FTPHost,
|
||||||
|
port: port,
|
||||||
|
baseDir: config.FTPDirectory,
|
||||||
|
client: conn,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a file via FTP.
|
||||||
|
func (f *FTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to open local file: %w", err)
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
// Create parent directory if needed
|
||||||
|
remoteDir := filepath.Dir(fullRemotePath)
|
||||||
|
remoteDir = normalizePathSeparators(remoteDir)
|
||||||
|
|
||||||
|
if remoteDir != "." && remoteDir != "/" {
|
||||||
|
f.ensureDirectory(remoteDir)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload file
|
||||||
|
if err := f.client.Stor(fullRemotePath, src); err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to upload file %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file via FTP.
|
||||||
|
func (f *FTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
// Get the file from FTP server
|
||||||
|
resp, err := f.client.Retr(fullRemotePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to retrieve file %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
defer resp.Close()
|
||||||
|
|
||||||
|
// Create local file
|
||||||
|
dst, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
// Copy data
|
||||||
|
if _, err := io.Copy(dst, resp); err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to transfer data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists files via FTP with the given prefix.
|
||||||
|
func (f *FTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
items := make([]Item, 0)
|
||||||
|
|
||||||
|
// Normalize the base directory
|
||||||
|
baseDir := normalizePathSeparators(f.baseDir)
|
||||||
|
if baseDir == "" {
|
||||||
|
baseDir = "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk the directory
|
||||||
|
if err := f.walkDirectory(baseDir, prefix, &items); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkDirectory recursively walks FTP directory structure.
|
||||||
|
func (f *FTPStorage) walkDirectory(dir string, prefix string, items *[]Item) error {
|
||||||
|
entries, err := f.client.List(dir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to list directory %s: %w", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
// Get relative path
|
||||||
|
var relPath string
|
||||||
|
if f.baseDir == "" || f.baseDir == "/" {
|
||||||
|
relPath = filepath.Join(strings.TrimPrefix(dir, "/"), entry.Name)
|
||||||
|
} else {
|
||||||
|
fullPath := filepath.Join(dir, entry.Name)
|
||||||
|
relPath = strings.TrimPrefix(fullPath, f.baseDir)
|
||||||
|
relPath = strings.TrimPrefix(relPath, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize separators for comparison
|
||||||
|
relPath = normalizePathSeparators(relPath)
|
||||||
|
|
||||||
|
// Skip if doesn't match prefix
|
||||||
|
if prefix != "" && !strings.HasPrefix(relPath, prefix) {
|
||||||
|
// For directories, we might need to walk them to find matching files
|
||||||
|
if entry.Type == ftp.EntryTypeFolder {
|
||||||
|
fullPath := filepath.Join(dir, entry.Name)
|
||||||
|
fullPath = normalizePathSeparators(fullPath)
|
||||||
|
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
|
||||||
|
// Log but don't fail on subdirectory errors
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
item := Item{
|
||||||
|
Key: relPath,
|
||||||
|
ModifiedTime: entry.Time,
|
||||||
|
Size: int64(entry.Size),
|
||||||
|
IsDirectory: entry.Type == ftp.EntryTypeFolder,
|
||||||
|
}
|
||||||
|
*items = append(*items, item)
|
||||||
|
|
||||||
|
// Recursively walk directories
|
||||||
|
if entry.Type == ftp.EntryTypeFolder {
|
||||||
|
fullPath := filepath.Join(dir, entry.Name)
|
||||||
|
fullPath = normalizePathSeparators(fullPath)
|
||||||
|
if err := f.walkDirectory(fullPath, prefix, items); err != nil {
|
||||||
|
// Log but don't fail on subdirectory errors
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes a file via FTP.
|
||||||
|
func (f *FTPStorage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
fullRemotePath := filepath.Join(f.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
if err := f.client.Delete(fullRemotePath); err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to remove %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources.
|
||||||
|
func (f *FTPStorage) Close() error {
|
||||||
|
if err := f.client.Quit(); err != nil {
|
||||||
|
return fmt.Errorf("ftp: failed to close connection: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureDirectory creates a directory path if it doesn't exist.
|
||||||
|
func (f *FTPStorage) ensureDirectory(path string) {
|
||||||
|
parts := strings.Split(path, "/")
|
||||||
|
current := ""
|
||||||
|
|
||||||
|
for _, part := range parts {
|
||||||
|
if part == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if current == "" {
|
||||||
|
current = part
|
||||||
|
} else {
|
||||||
|
current = current + "/" + part
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to create directory (ignore errors if it already exists)
|
||||||
|
_ = f.client.MakeDir(current)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
156
gcs.go
Normal file
156
gcs.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"cloud.google.com/go/storage"
|
||||||
|
"google.golang.org/api/iterator"
|
||||||
|
"google.golang.org/api/option"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GCSStorage implements Storage interface for Google Cloud Storage.
|
||||||
|
type GCSStorage struct {
|
||||||
|
client *storage.Client
|
||||||
|
bucket string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGCSStorage creates a new Google Cloud Storage instance.
|
||||||
|
func NewGCSStorage(config Config) (*GCSStorage, error) {
|
||||||
|
if err := validateConfig(config, map[string]string{
|
||||||
|
"bucket": config.GCSBucket,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("gcs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
options := make([]option.ClientOption, 0)
|
||||||
|
|
||||||
|
if config.GCSEndpoint != "" {
|
||||||
|
options = append(options, option.WithEndpoint(config.GCSEndpoint))
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.GCSCredentialsFile != "" {
|
||||||
|
options = append(options, option.WithCredentialsFile(config.GCSCredentialsFile))
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := storage.NewClient(context.Background(), options...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gcs: failed to create client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &GCSStorage{
|
||||||
|
client: client,
|
||||||
|
bucket: config.GCSBucket,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a file to GCS.
|
||||||
|
func (g *GCSStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to open local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("gcs: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewWriter(ctx)
|
||||||
|
defer func(obj *storage.Writer) {
|
||||||
|
err = obj.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("gcs: failed to close object writer: %v\n", err)
|
||||||
|
}
|
||||||
|
}(obj)
|
||||||
|
|
||||||
|
if _, err := io.Copy(obj, file); err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to write data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = obj.Close(); err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to finalize upload: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file from GCS.
|
||||||
|
func (g *GCSStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
file, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("gcs: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
obj, err := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath)).NewReader(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to create reader: %w", err)
|
||||||
|
}
|
||||||
|
defer func(obj *storage.Reader) {
|
||||||
|
err = obj.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("gcs: failed to close object reader: %v\n", err)
|
||||||
|
}
|
||||||
|
}(obj)
|
||||||
|
|
||||||
|
if _, err = io.Copy(file, obj); err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to read data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists objects in GCS with the given prefix.
|
||||||
|
func (g *GCSStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
items := make([]Item, 0)
|
||||||
|
|
||||||
|
query := &storage.Query{
|
||||||
|
Prefix: normalizePathSeparators(prefix),
|
||||||
|
}
|
||||||
|
|
||||||
|
it := g.client.Bucket(g.bucket).Objects(ctx, query)
|
||||||
|
|
||||||
|
for {
|
||||||
|
attrs, err := it.Next()
|
||||||
|
if err == iterator.Done {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gcs: failed to iterate objects: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
items = append(items, Item{
|
||||||
|
Key: attrs.Name,
|
||||||
|
ModifiedTime: attrs.Updated,
|
||||||
|
Size: attrs.Size,
|
||||||
|
IsDirectory: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes an object from GCS.
|
||||||
|
func (g *GCSStorage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
obj := g.client.Bucket(g.bucket).Object(normalizePathSeparators(remotePath))
|
||||||
|
|
||||||
|
if err := obj.Delete(ctx); err != nil {
|
||||||
|
return fmt.Errorf("gcs: failed to remove object from bucket %s: %w", g.bucket, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources.
|
||||||
|
func (g *GCSStorage) Close() error {
|
||||||
|
return g.client.Close()
|
||||||
|
}
|
||||||
67
go.mod
67
go.mod
@@ -1,22 +1,67 @@
|
|||||||
module github.com/jkaninda/go-storage
|
module github.com/jkaninda/go-storage
|
||||||
|
|
||||||
go 1.21.0
|
go 1.24.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/spf13/pflag v1.0.5
|
cloud.google.com/go/storage v1.60.0
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||||
|
github.com/aws/aws-sdk-go v1.55.7
|
||||||
|
github.com/bramvdbogaerde/go-scp v1.5.0
|
||||||
|
github.com/jlaffaye/ftp v0.2.0
|
||||||
|
github.com/pkg/sftp v1.13.10
|
||||||
|
golang.org/x/crypto v0.47.0
|
||||||
|
google.golang.org/api v0.267.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/aws/aws-sdk-go v1.55.3 // indirect
|
cel.dev/expr v0.24.0 // indirect
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0 // indirect
|
cloud.google.com/go v0.123.0 // indirect
|
||||||
|
cloud.google.com/go/auth v0.18.1 // indirect
|
||||||
|
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||||
|
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||||
|
cloud.google.com/go/iam v1.5.3 // indirect
|
||||||
|
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||||
|
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||||
|
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||||
|
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||||
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
|
github.com/google/s2a-go v0.1.9 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||||
|
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
github.com/hpcloud/tail v1.0.0 // indirect
|
|
||||||
github.com/jlaffaye/ftp v0.2.0 // indirect
|
|
||||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
github.com/kr/fs v0.1.0 // indirect
|
||||||
golang.org/x/crypto v0.18.0 // indirect
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||||
golang.org/x/sys v0.22.0 // indirect
|
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||||
gopkg.in/fsnotify.v1 v1.4.7 // indirect
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
|
||||||
|
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||||
|
golang.org/x/net v0.49.0 // indirect
|
||||||
|
golang.org/x/oauth2 v0.35.0 // indirect
|
||||||
|
golang.org/x/sync v0.19.0 // indirect
|
||||||
|
golang.org/x/sys v0.40.0 // indirect
|
||||||
|
golang.org/x/text v0.33.0 // indirect
|
||||||
|
golang.org/x/time v0.14.0 // indirect
|
||||||
|
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||||
|
google.golang.org/grpc v1.78.0 // indirect
|
||||||
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
198
go.sum
198
go.sum
@@ -1,68 +1,170 @@
|
|||||||
github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
|
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
|
||||||
github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
|
||||||
|
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||||
|
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||||
|
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||||
|
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||||
|
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||||
|
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||||
|
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||||
|
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||||
|
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
||||||
|
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
||||||
|
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||||
|
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||||
|
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
||||||
|
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
||||||
|
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
||||||
|
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
||||||
|
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
|
||||||
|
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
|
||||||
|
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
||||||
|
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
||||||
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
||||||
|
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
|
||||||
|
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
||||||
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
|
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
|
||||||
|
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
||||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
|
||||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
|
||||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
|
||||||
|
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||||
|
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||||
|
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||||
|
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||||
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
|
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||||
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
|
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||||
|
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||||
|
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||||
|
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||||
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||||
|
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
||||||
|
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||||
|
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||||
|
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
|
||||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
|
||||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||||
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
|
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
|
||||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
|
||||||
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0=
|
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
|
||||||
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||||
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
||||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
|
||||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
|
||||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
|
||||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
|
||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
|
||||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
|
||||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
||||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
||||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
|
||||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
|
||||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
|
||||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
|
||||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
|
||||||
|
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
|
||||||
|
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
||||||
|
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
||||||
|
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||||
|
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||||
|
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||||
|
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||||
|
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||||
|
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||||
|
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||||
|
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
|
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
|
||||||
|
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
|
||||||
|
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||||
|
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||||
|
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||||
|
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||||
|
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||||
|
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||||
|
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||||
|
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||||
|
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||||
|
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||||
|
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
|
||||||
|
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
|
||||||
|
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||||
|
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
|
||||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
348
local.go
Normal file
348
local.go
Normal file
@@ -0,0 +1,348 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LocalStorage implements the Storage interface using the local filesystem.
|
||||||
|
// It is safe for concurrent use as it relies on filesystem operations which
|
||||||
|
// are handled by the OS.
|
||||||
|
type LocalStorage struct {
|
||||||
|
basePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLocalStorage creates a new LocalStorage instance with the given base directory.
|
||||||
|
// The base directory will be created if it doesn't exist.
|
||||||
|
func NewLocalStorage(basePath string) (*LocalStorage, error) {
|
||||||
|
absPath, err := filepath.Abs(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to resolve absolute path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(absPath, 0755); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create base directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &LocalStorage{
|
||||||
|
basePath: absPath,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolvePath safely resolves a remote path to an absolute local path,
|
||||||
|
// ensuring it stays within the base directory.
|
||||||
|
func (s *LocalStorage) resolvePath(remotePath string) (string, error) {
|
||||||
|
// Clean the path to remove any .. or . components
|
||||||
|
cleanPath := filepath.Clean(remotePath)
|
||||||
|
|
||||||
|
// Remove leading slashes to make it relative
|
||||||
|
cleanPath = strings.TrimPrefix(cleanPath, string(filepath.Separator))
|
||||||
|
|
||||||
|
// Join with base path
|
||||||
|
fullPath := filepath.Join(s.basePath, cleanPath)
|
||||||
|
|
||||||
|
// Verify the resolved path is still within basePath
|
||||||
|
if !strings.HasPrefix(fullPath, s.basePath) {
|
||||||
|
return "", fmt.Errorf("path traversal attempt detected: %s", remotePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fullPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a local file to the storage with the given target name.
|
||||||
|
func (s *LocalStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
targetPath, err := s.resolvePath(remotePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open source file
|
||||||
|
srcFile, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open source file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Create parent directories if needed
|
||||||
|
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create parent directories: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create destination file
|
||||||
|
dstFile, err := os.Create(targetPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
// Copy with context cancellation checks
|
||||||
|
buf := make([]byte, 32*1024) // 32KB buffer
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Clean up partial file on cancellation
|
||||||
|
os.Remove(targetPath)
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
n, readErr := srcFile.Read(buf)
|
||||||
|
if n > 0 {
|
||||||
|
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
|
||||||
|
os.Remove(targetPath)
|
||||||
|
return fmt.Errorf("failed to write to destination: %w", writeErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if readErr == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
os.Remove(targetPath)
|
||||||
|
return fmt.Errorf("failed to read source file: %w", readErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file from storage to a local path.
|
||||||
|
func (s *LocalStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath, err := s.resolvePath(remotePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open source file
|
||||||
|
srcFile, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open remote file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Create parent directories for local path if needed
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create local parent directories: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create destination file
|
||||||
|
dstFile, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
// Copy with context cancellation checks
|
||||||
|
buf := make([]byte, 32*1024) // 32KB buffer
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Clean up partial file on cancellation
|
||||||
|
os.Remove(localPath)
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
n, readErr := srcFile.Read(buf)
|
||||||
|
if n > 0 {
|
||||||
|
if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil {
|
||||||
|
os.Remove(localPath)
|
||||||
|
return fmt.Errorf("failed to write to local file: %w", writeErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if readErr == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
os.Remove(localPath)
|
||||||
|
return fmt.Errorf("failed to read remote file: %w", readErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns all items in the storage that match the given prefix.
|
||||||
|
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
var items []Item
|
||||||
|
|
||||||
|
// Clean and resolve the prefix path
|
||||||
|
searchPath := s.basePath
|
||||||
|
cleanPrefix := strings.TrimPrefix(filepath.Clean(prefix), string(filepath.Separator))
|
||||||
|
|
||||||
|
if cleanPrefix != "" && cleanPrefix != "." {
|
||||||
|
searchPath = filepath.Join(s.basePath, cleanPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the search path exists
|
||||||
|
info, err := os.Stat(searchPath)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// If the exact path doesn't exist, try to find files with this prefix
|
||||||
|
// by walking the parent directory
|
||||||
|
parentDir := filepath.Dir(searchPath)
|
||||||
|
baseName := filepath.Base(searchPath)
|
||||||
|
|
||||||
|
if _, err := os.Stat(parentDir); os.IsNotExist(err) {
|
||||||
|
return items, nil // Return empty list if parent doesn't exist
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(parentDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.HasPrefix(entry.Name(), baseName) {
|
||||||
|
fullPath := filepath.Join(parentDir, entry.Name())
|
||||||
|
item, err := s.fileInfoToItem(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
continue // Skip files we can't stat
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to stat path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If it's a file, return just that file
|
||||||
|
if !info.IsDir() {
|
||||||
|
item, err := s.fileInfoToItem(searchPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []Item{item}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk the directory
|
||||||
|
err = filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil // Skip entries with errors
|
||||||
|
}
|
||||||
|
|
||||||
|
item, err := s.fileInfoToItem(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil // Skip files we can't stat
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to walk directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fileInfoToItem converts a file path to an Item.
|
||||||
|
func (s *LocalStorage) fileInfoToItem(path string) (Item, error) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return Item{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get relative path from base
|
||||||
|
relPath, err := filepath.Rel(s.basePath, path)
|
||||||
|
if err != nil {
|
||||||
|
relPath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize to forward slashes for consistency
|
||||||
|
relPath = filepath.ToSlash(relPath)
|
||||||
|
|
||||||
|
size := info.Size()
|
||||||
|
if info.IsDir() {
|
||||||
|
size = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return Item{
|
||||||
|
Key: relPath,
|
||||||
|
ModifiedTime: info.ModTime(),
|
||||||
|
Size: size,
|
||||||
|
IsDirectory: info.IsDir(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes a file from storage.
|
||||||
|
func (s *LocalStorage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
targetPath, err := s.resolvePath(remotePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if file exists
|
||||||
|
info, err := os.Stat(targetPath)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("file not found: %s", remotePath)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to stat file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove file or directory
|
||||||
|
if info.IsDir() {
|
||||||
|
if err := os.RemoveAll(targetPath); err != nil {
|
||||||
|
return fmt.Errorf("failed to remove directory: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := os.Remove(targetPath); err != nil {
|
||||||
|
return fmt.Errorf("failed to remove file: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases any resources held by the storage implementation.
|
||||||
|
// For LocalStorage, this is a no-op as there are no persistent resources.
|
||||||
|
func (s *LocalStorage) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BasePath returns the base directory path of the storage.
|
||||||
|
func (s *LocalStorage) BasePath() string {
|
||||||
|
return s.basePath
|
||||||
|
}
|
||||||
161
pkg/azure/azure.go
Normal file
161
pkg/azure/azure.go
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package azure
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||||
|
"github.com/jkaninda/go-storage/pkg"
|
||||||
|
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type azureStorage struct {
|
||||||
|
*pkg.Backend
|
||||||
|
client *azblob.Client
|
||||||
|
containerName string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
AccountName string
|
||||||
|
AccountKey string
|
||||||
|
ContainerName string
|
||||||
|
LocalPath string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// createClient creates FTP Client
|
||||||
|
func createClient(conf Config) (*azblob.Client, error) {
|
||||||
|
// Create the service URL
|
||||||
|
credential, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create service URL")
|
||||||
|
}
|
||||||
|
serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName)
|
||||||
|
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create client")
|
||||||
|
}
|
||||||
|
return client, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorage creates new Storage
|
||||||
|
func NewStorage(conf Config) (pkg.Storage, error) {
|
||||||
|
client, err := createClient(conf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &azureStorage{
|
||||||
|
client: client,
|
||||||
|
containerName: conf.ContainerName,
|
||||||
|
Backend: &pkg.Backend{
|
||||||
|
RemotePath: conf.RemotePath,
|
||||||
|
LocalPath: conf.LocalPath,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies file to Azure Blob Storage
|
||||||
|
func (s azureStorage) Copy(fileName string) error {
|
||||||
|
filePath := filepath.Join(s.LocalPath, fileName)
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
blobName := filepath.Join(s.RemotePath, fileName)
|
||||||
|
|
||||||
|
// Get the container client
|
||||||
|
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
|
||||||
|
|
||||||
|
// Get the blob client
|
||||||
|
blobClient := containerClient.NewBlockBlobClient(blobName)
|
||||||
|
|
||||||
|
// Upload the file
|
||||||
|
_, err = blobClient.UploadFile(context.Background(), file, nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to upload file: %v", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom copies a file from Azure Blob Storage to local storage
|
||||||
|
func (s azureStorage) CopyFrom(blobName string) error {
|
||||||
|
|
||||||
|
// Get the container client
|
||||||
|
containerClient := s.client.ServiceClient().NewContainerClient(s.containerName)
|
||||||
|
|
||||||
|
// Get the blob client
|
||||||
|
blobClient := containerClient.NewBlockBlobClient(filepath.Join(s.RemotePath, blobName))
|
||||||
|
|
||||||
|
// Download the blob
|
||||||
|
downloadResponse, err := blobClient.DownloadStream(context.Background(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the file to save the blob data
|
||||||
|
downloadFile, err := os.Create(filepath.Join(s.LocalPath, blobName))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func(downloadFile *os.File) {
|
||||||
|
err := downloadFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to close file: %v", err)
|
||||||
|
}
|
||||||
|
}(downloadFile)
|
||||||
|
|
||||||
|
// Write the blob data to the file
|
||||||
|
_, err = io.Copy(downloadFile, downloadResponse.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to write blob to file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune deletes old backup created more than specified days
|
||||||
|
func (s azureStorage) Prune(retentionDays int) error {
|
||||||
|
fmt.Println("Deleting old backup from a remote server is not implemented yet")
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the storage name
|
||||||
|
func (s azureStorage) Name() string {
|
||||||
|
return "azure"
|
||||||
|
}
|
||||||
@@ -1,3 +1,27 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
package ftp
|
package ftp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,14 +44,14 @@ type Config struct {
|
|||||||
Host string
|
Host string
|
||||||
User string
|
User string
|
||||||
Password string
|
Password string
|
||||||
Port string
|
Port int
|
||||||
LocalPath string
|
LocalPath string
|
||||||
RemotePath string
|
RemotePath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// createClient creates FTP Client
|
// createClient creates FTP Client
|
||||||
func createClient(conf Config) (*ftp.ServerConn, error) {
|
func createClient(conf Config) (*ftp.ServerConn, error) {
|
||||||
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
|
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%d", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
|
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
|
||||||
}
|
}
|
||||||
@@ -58,14 +82,24 @@ func NewStorage(conf Config) (pkg.Storage, error) {
|
|||||||
// Copy copies file to the remote server
|
// Copy copies file to the remote server
|
||||||
func (s ftpStorage) Copy(fileName string) error {
|
func (s ftpStorage) Copy(fileName string) error {
|
||||||
ftpClient := s.client
|
ftpClient := s.client
|
||||||
defer ftpClient.Quit()
|
defer func(ftpClient *ftp.ServerConn) {
|
||||||
|
err := ftpClient.Quit()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(ftpClient)
|
||||||
|
|
||||||
filePath := filepath.Join(s.LocalPath, fileName)
|
filePath := filepath.Join(s.LocalPath, fileName)
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||||
err = ftpClient.Stor(remoteFilePath, file)
|
err = ftpClient.Stor(remoteFilePath, file)
|
||||||
@@ -80,21 +114,36 @@ func (s ftpStorage) Copy(fileName string) error {
|
|||||||
func (s ftpStorage) CopyFrom(fileName string) error {
|
func (s ftpStorage) CopyFrom(fileName string) error {
|
||||||
ftpClient := s.client
|
ftpClient := s.client
|
||||||
|
|
||||||
defer ftpClient.Quit()
|
defer func(ftpClient *ftp.ServerConn) {
|
||||||
|
err := ftpClient.Quit()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(ftpClient)
|
||||||
|
|
||||||
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||||
r, err := ftpClient.Retr(remoteFilePath)
|
r, err := ftpClient.Retr(remoteFilePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
|
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
|
||||||
}
|
}
|
||||||
defer r.Close()
|
defer func(r *ftp.Response) {
|
||||||
|
err := r.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(r)
|
||||||
|
|
||||||
localFilePath := filepath.Join(s.LocalPath, fileName)
|
localFilePath := filepath.Join(s.LocalPath, fileName)
|
||||||
outFile, err := os.Create(localFilePath)
|
outFile, err := os.Create(localFilePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
|
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
|
||||||
}
|
}
|
||||||
defer outFile.Close()
|
defer func(outFile *os.File) {
|
||||||
|
err := outFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(outFile)
|
||||||
|
|
||||||
_, err = io.Copy(outFile, r)
|
_, err = io.Copy(outFile, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,3 +1,27 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
package local
|
package local
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -92,7 +116,12 @@ func copyFile(src, dst string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer in.Close()
|
defer func(in *os.File) {
|
||||||
|
err := in.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(in)
|
||||||
|
|
||||||
out, err := os.Create(dst)
|
out, err := os.Create(dst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -101,7 +130,10 @@ func copyFile(src, dst string) error {
|
|||||||
|
|
||||||
_, err = io.Copy(out, in)
|
_, err = io.Copy(out, in)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out.Close()
|
err := out.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return out.Close()
|
return out.Close()
|
||||||
|
|||||||
@@ -45,7 +45,12 @@ func createFile(fileName, content string) ([]byte, error) {
|
|||||||
fmt.Println("Error creating file:", err)
|
fmt.Println("Error creating file:", err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
// Write the message to the file
|
// Write the message to the file
|
||||||
_, err = file.WriteString(content)
|
_, err = file.WriteString(content)
|
||||||
|
|||||||
50
pkg/s3/s3.go
50
pkg/s3/s3.go
@@ -1,3 +1,27 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
package s3
|
package s3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -70,7 +94,12 @@ func (s s3Storage) Copy(fileName string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
fileInfo, err := file.Stat()
|
fileInfo, err := file.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -78,7 +107,10 @@ func (s s3Storage) Copy(fileName string) error {
|
|||||||
}
|
}
|
||||||
objectKey := filepath.Join(s.RemotePath, fileName)
|
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||||
buffer := make([]byte, fileInfo.Size())
|
buffer := make([]byte, fileInfo.Size())
|
||||||
file.Read(buffer)
|
_, err = file.Read(buffer)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
fileBytes := bytes.NewReader(buffer)
|
fileBytes := bytes.NewReader(buffer)
|
||||||
fileType := http.DetectContentType(buffer)
|
fileType := http.DetectContentType(buffer)
|
||||||
|
|
||||||
@@ -102,7 +134,13 @@ func (s s3Storage) CopyFrom(fileName string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error closing file: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
objectKey := filepath.Join(s.RemotePath, fileName)
|
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||||
|
|
||||||
@@ -140,16 +178,16 @@ func (s s3Storage) Prune(retentionDays int) error {
|
|||||||
Key: object.Key,
|
Key: object.Key,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to delete object %s: %v", *object.Key, err)
|
fmt.Printf("failed to delete object %s: %v\n", *object.Key, err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("Deleted object %s", *object.Key)
|
fmt.Printf("Deleted object %s\n", *object.Key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return !lastPage
|
return !lastPage
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to list objects: %v", err)
|
return fmt.Errorf("failed to list objects: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,3 +1,27 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
package ssh
|
package ssh
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -22,7 +46,7 @@ type Config struct {
|
|||||||
Host string
|
Host string
|
||||||
User string
|
User string
|
||||||
Password string
|
Password string
|
||||||
Port string
|
Port int
|
||||||
IdentifyFile string
|
IdentifyFile string
|
||||||
LocalPath string
|
LocalPath string
|
||||||
RemotePath string
|
RemotePath string
|
||||||
@@ -30,15 +54,15 @@ type Config struct {
|
|||||||
|
|
||||||
// createClient creates SSH Client
|
// createClient creates SSH Client
|
||||||
func createClient(conf Config) (scp.Client, error) {
|
func createClient(conf Config) (scp.Client, error) {
|
||||||
if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) {
|
if _, err := os.Stat(conf.IdentifyFile); !os.IsNotExist(err) {
|
||||||
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
|
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
|
||||||
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
|
||||||
} else {
|
} else {
|
||||||
if conf.Password == "" {
|
if conf.Password == "" {
|
||||||
return scp.Client{}, errors.New("ssh password required")
|
return scp.Client{}, errors.New("ssh password required")
|
||||||
}
|
}
|
||||||
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
|
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
|
||||||
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
return scp.NewClient(fmt.Sprintf("%s:%d", conf.Host, conf.Port), &clientConfig), err
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -97,7 +121,12 @@ func (s sshStorage) CopyFrom(fileName string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.New("couldn't open the output file")
|
return errors.New("couldn't open the output file")
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer func(file *os.File) {
|
||||||
|
err := file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
|
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
|
||||||
|
|
||||||
|
|||||||
1
pkg/ssh/ssh_test.go
Normal file
1
pkg/ssh/ssh_test.go
Normal file
@@ -0,0 +1 @@
|
|||||||
|
package ssh
|
||||||
@@ -1,3 +1,27 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
package pkg
|
package pkg
|
||||||
|
|
||||||
type Storage interface {
|
type Storage interface {
|
||||||
|
|||||||
187
s3.go
Normal file
187
s3.go
Normal file
@@ -0,0 +1,187 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
|
)
|
||||||
|
|
||||||
|
// S3Storage implements Storage interface for Amazon S3.
|
||||||
|
type S3Storage struct {
|
||||||
|
session *session.Session
|
||||||
|
bucket string
|
||||||
|
region string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewS3Storage creates a new S3 storage instance.
|
||||||
|
func NewS3Storage(config Config) (*S3Storage, error) {
|
||||||
|
if err := validateConfig(config, map[string]string{
|
||||||
|
"bucket": config.S3Bucket,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("s3: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
awsConfig := aws.NewConfig()
|
||||||
|
|
||||||
|
if config.S3Region != "" {
|
||||||
|
awsConfig = awsConfig.WithRegion(config.S3Region)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.S3KeyID != "" && config.S3Secret != "" {
|
||||||
|
awsConfig = awsConfig.WithCredentials(
|
||||||
|
credentials.NewStaticCredentials(config.S3KeyID, config.S3Secret, ""),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.S3Endpoint != "" {
|
||||||
|
awsConfig = awsConfig.WithEndpoint(config.S3Endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.S3ForcePath {
|
||||||
|
awsConfig = awsConfig.WithS3ForcePathStyle(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.S3DisableTLS {
|
||||||
|
awsConfig = awsConfig.WithDisableSSL(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionOpts := session.Options{
|
||||||
|
Config: *awsConfig,
|
||||||
|
SharedConfigState: session.SharedConfigEnable,
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.S3Profile != "" {
|
||||||
|
sessionOpts.Profile = config.S3Profile
|
||||||
|
}
|
||||||
|
|
||||||
|
sess, err := session.NewSessionWithOptions(sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("s3: failed to create session: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &S3Storage{
|
||||||
|
session: sess,
|
||||||
|
bucket: config.S3Bucket,
|
||||||
|
region: config.S3Region,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a file to S3.
|
||||||
|
func (s *S3Storage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("s3: failed to open local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("s3: warning: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
uploader := s3manager.NewUploader(s.session)
|
||||||
|
|
||||||
|
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||||
|
Body: file,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("s3: failed to upload to bucket %s: %w", s.bucket, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file from S3.
|
||||||
|
func (s *S3Storage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
file, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("s3: failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer func(file *os.File) {
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("s3: warning: failed to close file: %v\n", err)
|
||||||
|
}
|
||||||
|
}(file)
|
||||||
|
|
||||||
|
downloader := s3manager.NewDownloader(s.session)
|
||||||
|
|
||||||
|
_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("s3: failed to download from bucket %s: %w", s.bucket, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists objects in S3 with the given prefix.
|
||||||
|
func (s *S3Storage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
svc := s3.New(s.session)
|
||||||
|
items := make([]Item, 0)
|
||||||
|
|
||||||
|
var continuationToken *string
|
||||||
|
|
||||||
|
for {
|
||||||
|
input := &s3.ListObjectsV2Input{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Prefix: aws.String(normalizePathSeparators(prefix)),
|
||||||
|
ContinuationToken: continuationToken,
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := svc.ListObjectsV2WithContext(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("s3: failed to list objects in bucket %s: %w", s.bucket, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, obj := range resp.Contents {
|
||||||
|
items = append(items, Item{
|
||||||
|
Key: *obj.Key,
|
||||||
|
ModifiedTime: *obj.LastModified,
|
||||||
|
Size: *obj.Size,
|
||||||
|
IsDirectory: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
continuationToken = resp.NextContinuationToken
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes an object from S3.
|
||||||
|
func (s *S3Storage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
svc := s3.New(s.session)
|
||||||
|
|
||||||
|
_, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(normalizePathSeparators(remotePath)),
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("s3: failed to remove object from bucket %s: %w", s.bucket, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources.
|
||||||
|
func (s *S3Storage) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
350
sftp.go
Normal file
350
sftp.go
Normal file
@@ -0,0 +1,350 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/pkg/sftp"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"golang.org/x/crypto/ssh/agent"
|
||||||
|
"golang.org/x/crypto/ssh/knownhosts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SFTPStorage implements Storage interface for SFTP.
|
||||||
|
type SFTPStorage struct {
|
||||||
|
host string
|
||||||
|
baseDir string
|
||||||
|
sshClient *ssh.Client
|
||||||
|
client *sftp.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSFTPStorage creates a new SFTP storage instance.
|
||||||
|
func NewSFTPStorage(config Config) (*SFTPStorage, error) {
|
||||||
|
if err := validateConfig(config, map[string]string{
|
||||||
|
"host": config.SFTPHost,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("sftp: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
port := config.SFTPPort
|
||||||
|
if port == "" {
|
||||||
|
port = "22"
|
||||||
|
}
|
||||||
|
|
||||||
|
username := config.SFTPUsername
|
||||||
|
if username == "" {
|
||||||
|
currentUser, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("sftp: failed to get current user: %w", err)
|
||||||
|
}
|
||||||
|
username = currentUser.Username
|
||||||
|
}
|
||||||
|
|
||||||
|
password := config.SFTPPassword
|
||||||
|
if password == "" {
|
||||||
|
password = os.Getenv("PGBK_SSH_PASS")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build authentication methods
|
||||||
|
authMethods := make([]ssh.AuthMethod, 0)
|
||||||
|
|
||||||
|
// Password authentication (if no identity file is specified)
|
||||||
|
if config.SFTPIdentityFile == "" && password != "" {
|
||||||
|
authMethods = append(authMethods, ssh.Password(password))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Public key authentication
|
||||||
|
signers, err := getSSHSigners(config.SFTPIdentityFile, password)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("sftp: %w", err)
|
||||||
|
}
|
||||||
|
if len(signers) > 0 {
|
||||||
|
authMethods = append(authMethods, ssh.PublicKeys(signers...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build SSH client config
|
||||||
|
sshConfig := &ssh.ClientConfig{
|
||||||
|
User: username,
|
||||||
|
Auth: authMethods,
|
||||||
|
HostKeyCallback: getHostKeyCallback(config.SFTPIgnoreKnownHosts),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect to SSH server
|
||||||
|
hostPort := fmt.Sprintf("%s:%s", config.SFTPHost, port)
|
||||||
|
sshClient, err := ssh.Dial("tcp", hostPort, sshConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("sftp: failed to connect to %s: %w", hostPort, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open SFTP session
|
||||||
|
sftpClient, err := sftp.NewClient(sshClient)
|
||||||
|
if err != nil {
|
||||||
|
sshClient.Close()
|
||||||
|
return nil, fmt.Errorf("sftp: failed to open sftp session: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
baseDir := config.SFTPDirectory
|
||||||
|
if baseDir == "" {
|
||||||
|
wd, err := sftpClient.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
sftpClient.Close()
|
||||||
|
sshClient.Close()
|
||||||
|
return nil, fmt.Errorf("sftp: failed to get working directory: %w", err)
|
||||||
|
}
|
||||||
|
baseDir = wd
|
||||||
|
}
|
||||||
|
|
||||||
|
return &SFTPStorage{
|
||||||
|
host: config.SFTPHost,
|
||||||
|
baseDir: baseDir,
|
||||||
|
sshClient: sshClient,
|
||||||
|
client: sftpClient,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload uploads a file via SFTP.
|
||||||
|
func (s *SFTPStorage) Upload(ctx context.Context, localPath string, remotePath string) error {
|
||||||
|
src, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to open local file: %w", err)
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
// Create parent directory if needed
|
||||||
|
remoteDir := filepath.Dir(fullRemotePath)
|
||||||
|
remoteDir = normalizePathSeparators(remoteDir)
|
||||||
|
|
||||||
|
if remoteDir != "." && remoteDir != "/" {
|
||||||
|
if err := s.client.MkdirAll(remoteDir); err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to create directory %s: %w", remoteDir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dst, err := s.client.Create(fullRemotePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to create remote file %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(dst, src); err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to transfer data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download downloads a file via SFTP.
|
||||||
|
func (s *SFTPStorage) Download(ctx context.Context, remotePath string, localPath string) error {
|
||||||
|
dst, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to create local file: %w", err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
src, err := s.client.Open(fullRemotePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to open remote file %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(dst, src); err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to transfer data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists files via SFTP with the given prefix.
|
||||||
|
func (s *SFTPStorage) List(ctx context.Context, prefix string) ([]Item, error) {
|
||||||
|
items := make([]Item, 0)
|
||||||
|
|
||||||
|
baseDir := normalizePathSeparators(s.baseDir)
|
||||||
|
|
||||||
|
walker := s.client.Walk(baseDir)
|
||||||
|
for walker.Step() {
|
||||||
|
if err := walker.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("sftp: walk error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get relative path
|
||||||
|
path := walker.Path()
|
||||||
|
relPath, err := filepath.Rel(baseDir, path)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize separators for comparison
|
||||||
|
relPath = normalizePathSeparators(relPath)
|
||||||
|
|
||||||
|
if !strings.HasPrefix(relPath, prefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
stat := walker.Stat()
|
||||||
|
items = append(items, Item{
|
||||||
|
Key: relPath,
|
||||||
|
ModifiedTime: stat.ModTime(),
|
||||||
|
Size: stat.Size(),
|
||||||
|
IsDirectory: stat.IsDir(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes a file via SFTP.
|
||||||
|
func (s *SFTPStorage) Remove(ctx context.Context, remotePath string) error {
|
||||||
|
fullRemotePath := filepath.Join(s.baseDir, remotePath)
|
||||||
|
fullRemotePath = normalizePathSeparators(fullRemotePath)
|
||||||
|
|
||||||
|
if err := s.client.Remove(fullRemotePath); err != nil {
|
||||||
|
return fmt.Errorf("sftp: failed to remove %s: %w", fullRemotePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources.
|
||||||
|
func (s *SFTPStorage) Close() error {
|
||||||
|
if err := s.client.Close(); err != nil {
|
||||||
|
s.sshClient.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s.sshClient.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSSHSigners returns SSH signers from identity file and SSH agent.
|
||||||
|
func getSSHSigners(identityFile string, passphrase string) ([]ssh.Signer, error) {
|
||||||
|
signers := make([]ssh.Signer, 0)
|
||||||
|
|
||||||
|
// Load identity file if provided
|
||||||
|
if identityFile != "" {
|
||||||
|
path, err := expandHomeDir(identityFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to expand identity file path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keyData, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read identity file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
signer, err := ssh.ParsePrivateKey(keyData)
|
||||||
|
if err != nil {
|
||||||
|
var passErr *ssh.PassphraseMissingError
|
||||||
|
if errors.As(err, &passErr) {
|
||||||
|
signer, err = ssh.ParsePrivateKeyWithPassphrase(keyData, []byte(passphrase))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decrypt identity file: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("failed to parse identity file: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
signers = append(signers, signer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to get keys from SSH agent
|
||||||
|
socket := os.Getenv("SSH_AUTH_SOCK")
|
||||||
|
if socket != "" {
|
||||||
|
conn, err := net.Dial("unix", socket)
|
||||||
|
if err == nil {
|
||||||
|
agentClient := agent.NewClient(conn)
|
||||||
|
agentSigners, err := agentClient.Signers()
|
||||||
|
if err == nil {
|
||||||
|
signers = append(signers, agentSigners...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return signers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getHostKeyCallback returns appropriate host key callback.
|
||||||
|
func getHostKeyCallback(ignoreHostKey bool) ssh.HostKeyCallback {
|
||||||
|
if ignoreHostKey {
|
||||||
|
return ssh.InsecureIgnoreHostKey()
|
||||||
|
}
|
||||||
|
|
||||||
|
knownHostsFiles := make([]string, 0)
|
||||||
|
for _, p := range []string{"/etc/ssh/ssh_known_hosts", "~/.ssh/known_hosts"} {
|
||||||
|
path, err := expandHomeDir(p)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
knownHostsFiles = append(knownHostsFiles, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(knownHostsFiles) == 0 {
|
||||||
|
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||||
|
return fmt.Errorf("no known_hosts files found for host key verification")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
callback, err := knownhosts.New(knownHostsFiles...)
|
||||||
|
if err != nil {
|
||||||
|
return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||||
|
return fmt.Errorf("failed to load known_hosts: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// expandHomeDir expands ~ in file paths to home directory.
|
||||||
|
func expandHomeDir(path string) (string, error) {
|
||||||
|
if !strings.HasPrefix(path, "~") {
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.SplitN(path, "/", 2)
|
||||||
|
username := strings.TrimPrefix(parts[0], "~")
|
||||||
|
|
||||||
|
var homeDir string
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if username == "" {
|
||||||
|
homeDir, err = os.UserHomeDir()
|
||||||
|
if err != nil || homeDir == "" {
|
||||||
|
currentUser, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get home directory: %w", err)
|
||||||
|
}
|
||||||
|
homeDir = currentUser.HomeDir
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
userInfo, err := user.Lookup(username)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to lookup user %s: %w", username, err)
|
||||||
|
}
|
||||||
|
homeDir = userInfo.HomeDir
|
||||||
|
}
|
||||||
|
|
||||||
|
if homeDir == "" {
|
||||||
|
return "", fmt.Errorf("empty home directory")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(parts) == 1 {
|
||||||
|
return homeDir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(homeDir, parts[1]), nil
|
||||||
|
}
|
||||||
140
storage.go
Normal file
140
storage.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Storage defines the interface for remote file storage operations.
|
||||||
|
// All implementations must be safe for concurrent use.
|
||||||
|
type Storage interface {
|
||||||
|
// Upload uploads a local file to the remote storage with the given target name.
|
||||||
|
Upload(ctx context.Context, localPath string, remotePath string) error
|
||||||
|
|
||||||
|
// Download downloads a file from remote storage to a local path.
|
||||||
|
Download(ctx context.Context, remotePath string, localPath string) error
|
||||||
|
|
||||||
|
// List returns all items in the storage that match the given prefix.
|
||||||
|
List(ctx context.Context, prefix string) ([]Item, error)
|
||||||
|
|
||||||
|
// Remove deletes a file from remote storage.
|
||||||
|
Remove(ctx context.Context, remotePath string) error
|
||||||
|
|
||||||
|
// Close releases any resources held by the storage implementation.
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Item represents a file or directory in remote storage.
|
||||||
|
type Item struct {
|
||||||
|
Key string // The path/key of the item
|
||||||
|
ModifiedTime time.Time // Last modification time
|
||||||
|
Size int64 // Size in bytes (0 for directories)
|
||||||
|
IsDirectory bool // Whether this item is a directory
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds configuration options for creating storage instances.
|
||||||
|
type Config struct {
|
||||||
|
// S3 configuration
|
||||||
|
S3Region string
|
||||||
|
S3Bucket string
|
||||||
|
S3Profile string
|
||||||
|
S3KeyID string
|
||||||
|
S3Secret string
|
||||||
|
S3Endpoint string
|
||||||
|
S3ForcePath bool
|
||||||
|
S3DisableTLS bool
|
||||||
|
|
||||||
|
// B2 configuration
|
||||||
|
B2KeyID string
|
||||||
|
B2AppKey string
|
||||||
|
B2Bucket string
|
||||||
|
B2ConcurrentConnections int
|
||||||
|
B2ForcePath bool
|
||||||
|
|
||||||
|
// Google Cloud Storage configuration
|
||||||
|
GCSBucket string
|
||||||
|
GCSEndpoint string
|
||||||
|
GCSCredentialsFile string
|
||||||
|
|
||||||
|
// Azure configuration
|
||||||
|
AzureAccount string
|
||||||
|
AzureKey string
|
||||||
|
AzureContainer string
|
||||||
|
AzureEndpoint string
|
||||||
|
|
||||||
|
// SFTP configuration
|
||||||
|
SFTPHost string
|
||||||
|
SFTPPort string
|
||||||
|
SFTPUsername string
|
||||||
|
SFTPPassword string
|
||||||
|
SFTPDirectory string
|
||||||
|
SFTPIdentityFile string
|
||||||
|
SFTPIgnoreKnownHosts bool
|
||||||
|
|
||||||
|
// FTP configuration
|
||||||
|
FTPHost string
|
||||||
|
FTPPort string
|
||||||
|
FTPUsername string
|
||||||
|
FTPPassword string
|
||||||
|
FTPDirectory string
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageType represents the type of storage backend.
|
||||||
|
type StorageType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
StorageTypeS3 StorageType = "s3"
|
||||||
|
StorageTypeGCS StorageType = "gcs"
|
||||||
|
StorageTypeAzure StorageType = "azure"
|
||||||
|
StorageTypeSFTP StorageType = "sftp"
|
||||||
|
StorageTypeFTP StorageType = "ftp"
|
||||||
|
StorageTypeLocal StorageType = "local"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewStorage creates a new Storage instance based on the specified type and configuration.
|
||||||
|
func NewStorage(storageType StorageType, config Config) (Storage, error) {
|
||||||
|
switch storageType {
|
||||||
|
case StorageTypeS3:
|
||||||
|
return NewS3Storage(config)
|
||||||
|
case StorageTypeGCS:
|
||||||
|
return NewGCSStorage(config)
|
||||||
|
case StorageTypeAzure:
|
||||||
|
return NewAzureStorage(config)
|
||||||
|
case StorageTypeSFTP:
|
||||||
|
return NewSFTPStorage(config)
|
||||||
|
case StorageTypeFTP:
|
||||||
|
return NewFTPStorage(config)
|
||||||
|
case StorageTypeLocal:
|
||||||
|
return NewLocalStorage("/backups")
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizePathSeparators converts OS-specific path separators to forward slashes.
|
||||||
|
// This is necessary for cloud storage services which expect forward slashes.
|
||||||
|
func normalizePathSeparators(path string) string {
|
||||||
|
if os.PathSeparator == '/' {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
return strings.ReplaceAll(path, string(os.PathSeparator), "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateConfig checks if the required configuration fields are present.
|
||||||
|
func validateConfig(_ Config, requiredFields map[string]string) error {
|
||||||
|
var missing []string
|
||||||
|
for field, value := range requiredFields {
|
||||||
|
if value == "" {
|
||||||
|
missing = append(missing, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(missing) > 0 {
|
||||||
|
return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
41
storage_test.go
Normal file
41
storage_test.go
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
package go_storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Test helpers
|
||||||
|
func TestStorageInterface(t *testing.T) {
|
||||||
|
var _ Storage = (*S3Storage)(nil)
|
||||||
|
var _ Storage = (*GCSStorage)(nil)
|
||||||
|
var _ Storage = (*AzureStorage)(nil)
|
||||||
|
var _ Storage = (*SFTPStorage)(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextTimeout(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
config := Config{
|
||||||
|
S3Region: "us-east-1",
|
||||||
|
S3Bucket: "test-bucket",
|
||||||
|
}
|
||||||
|
|
||||||
|
s3, err := NewStorage(StorageTypeS3, config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create storage: %v", err)
|
||||||
|
}
|
||||||
|
defer func(s3 Storage) {
|
||||||
|
err = s3.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Failed to close storage: %v", err)
|
||||||
|
}
|
||||||
|
}(s3)
|
||||||
|
|
||||||
|
err = s3.Upload(ctx, "test.txt", "remote-test.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Upload failed (expected in test): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user