It's still RC version, but I tried using S3 object upload and download using AWS SDK for Go v2. https://github.com/aws/aws-sdk-go-v2
I don't usually touch go, but I wrote it in practice.
Go version is v1.15
$ go version
go version go1.15 linux/amd64
go mod init awssdkapp
go get -u github.com/aws/aws-sdk-go-v2
go get -u github.com/aws/aws-sdk-go-v2/config
go get -u github.com/aws/aws-sdk-go-v2/feature/s3/manager
go get -u github.com/aws/aws-sdk-go-v2/service/s3
touch main.go
mkdir myclient
touch myclient/myclient.go
Simply upload or download objects.
myclient/myclient.go
package myclient
import (
"context"
"io"
"log"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type MyS3Client struct {
downloader *manager.Downloader
uploader *manager.Uploader
client *s3.Client
}
func NewMyS3Client(cfg aws.Config) *MyS3Client {
client := s3.NewFromConfig(cfg)
downloader := manager.NewDownloader(client)
uploader := manager.NewUploader(client)
return &MyS3Client{
downloader: downloader,
uploader: uploader,
client: client,
}
}
//Method to download object
func (c *MyS3Client) UploadSingleObject(bucket, key string, reader io.Reader) {
_, err := c.uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: reader,
})
if err != nil {
log.Fatal(err)
}
log.Println("upload successed")
}
//Method to download object
func (c *MyS3Client) DownloadSingleObject(bucket, key, filename string) {
file, _ := os.Create(filename)
defer file.Close()
_, err := c.downloader.Download(context.Background(), file, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
log.Fatal(err)
}
log.Println("download successed")
}
main.go
package main
import (
"context"
"flag"
"log"
"strings"
"awsdkapp/myclient"
"github.com/aws/aws-sdk-go-v2/config"
)
func main() {
bucket := flag.String("bucket", "", "specify bucket name")
key := flag.String("key", "", "specify key name ")
flag.Parse()
if *bucket == "" || *key == "" {
log.Fatal("error invalid arugments")
}
//Creating a config
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
log.Fatal(err)
}
client := myclient.NewMyS3Client(cfg)
//Upload content
client.UploadSingleObject(*bucket, *key, strings.NewReader("hello, aws sdk"))
//Download as a file
client.DownloadSingleObject(*bucket, *key, "hello.txt")
}
If you try it, you should be able to upload/download it.
$ go run main.go -bucket the-chocomintkusoyaro-demo -key hoge
2020/12/30 23:05:32 upload successed
2020/12/30 23:05:32 download successed
$ ls hello.txt
hello.txt
$ cat hello.txt
hello, aws sdk
Generate a pagenator with NewListObjectsV2Paginator, get the object group for each page, and download and write each object.
myclient/myclient.go
...
//Download multiple objects
func (c *MyS3Client) DownloadMultipleObjects(bucket, prefix, localDir string) {
//Create a page nator
paginator := s3.NewListObjectsV2Paginator(c.client, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
})
for paginator.HasMorePages() {
page, err := paginator.NextPage(context.Background())
if err != nil {
log.Fatal(err)
}
for _, obj := range page.Contents {
if err != nil {
log.Fatal(err)
}
filepath := filepath.Join(localDir, *obj.Key)
c.DownloadSingleObject(bucket, *obj.Key, filepath)
}
}
log.Println("all file downloaded")
}
main.go
package main
import (
"context"
"flag"
"log"
"awsdkapp/myclient"
"github.com/aws/aws-sdk-go-v2/config"
)
func main() {
bucket := flag.String("bucket", "", "specify bucket name")
prefix := flag.String("prefix", "", "specify prefix name")
localDir := flag.String("localDir", "", "specify local dir ")
flag.Parse()
if *bucket == "" || *localDir == "" {
log.Fatal("error invalid arugments")
}
//Creating a config
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
log.Fatal(err)
}
client := myclient.NewMyS3Client(cfg)
//Download the object
client.DownloadMultipleObjects(*bucket, *prefix, *localDir)
}
I tried to imitate the Example of the document.
myclient/myclient.go
...
//Upload multiple objects at once
func (c *MyS3Client) UploadMultipleObjects(bucket, prefix, localDir string) {
fileWalker := make(chan string)
//Send the local file path to the channel
go func() {
err := filepath.Walk(localDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
//Send file path to channel
fileWalker <- path
}
return nil
})
if err != nil {
log.Fatal(err)
}
//Notify channel closure
close(fileWalker)
}()
//Receive file path from channel
for path := range fileWalker {
rel, err := filepath.Rel(localDir, path)
if err != nil {
log.Println("unable to get relative path", path, err)
continue
}
file, err := os.Open(path)
if err != nil {
log.Println("failed open file", path, err)
continue
}
defer file.Close()
key := filepath.Join(prefix, rel)
//Download the object
c.UploadSingleObject(bucket, key, file)
}
}
main.go
package main
import (
"context"
"flag"
"log"
"awsdkapp/myclient"
"github.com/aws/aws-sdk-go-v2/config"
)
func main() {
bucket := flag.String("bucket", "", "specify bucket name")
prefix := flag.String("prefix", "", "specify prefix name")
localDir := flag.String("localDir", "", "specify local dir ")
flag.Parse()
if *bucket == "" || *localDir == "" {
log.Fatal("error invalid arugments")
}
//Creating a config
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
log.Fatal(err)
}
client := myclient.NewMyS3Client(cfg)
//Upload objects in bulk
client.UploadMultipleObjects(*bucket, *prefix, *localDir)
}
Recommended Posts