s3dlmgr.go

// Uses the default AWS SDK Credentials;  e.g. via the environment
// AWS_REGION=region AWS_ACCESS_KEY_ID=key AWS_SECRET_ACCESS_KEY=secret
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func main() {

	var bucket string
	var key string
	var debug bool

	flag.StringVar(&bucket, "bucket", "", "s3 bucket")
	flag.StringVar(&key, "key", "", "s3 key (path)")
	flag.BoolVar(&debug, "debug", false, "show aws sdk debug output")

	flag.Parse()

	if len(bucket) == 0 || len(key) == 0 {
		flag.Usage()
		os.Exit(-1)
	}

	cfg := aws.NewConfig()
	if debug {
		log := log.New(os.Stderr, "", log.LstdFlags)
		cfg = cfg.WithLogger(
			aws.LoggerFunc(func(args ...interface{}) { log.Println(args) }),
		).WithLogLevel(aws.LogDebugWithSigning)
	}

	awsSession := session.New(cfg)

	fmt.Fprintf(os.Stderr, "attempting to download s3://%v/%v\n", bucket, key)

	start := time.Now()

	buff := &aws.WriteAtBuffer{}
	s3dl := s3manager.NewDownloader(awsSession)
	n, err := s3dl.Download(buff, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	n2, err := io.Copy(os.Stdout, bytes.NewReader(buff.Bytes()))

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}

	if n != n2 {
		fmt.Fprintf(os.Stderr, "bytes written (%v) != bytes received (%v)\n", n2, n)
		os.Exit(3)
	}

	fmt.Fprintf(os.Stderr, "wrote %v bytes to stdout in %v\n", n, time.Now().Sub(start))

}

Hi Joshua,
while looking how to solve a problem I have, I came across this gist. If I understand it correctly, downloading into the aws.WriteAtBuffer would result in having the complete object in memory? Am I right about that?

What I'm looking for, is an extended version of the WriteAtBuffer. To keep the memory footprint small, I would like to do this:
While the download is running, I read from the buffer and it returns all bytes until the next gap (missing part), the returned bytes will be removed from the start of the buffer. This will be repeated until the download is finished and the last bytes are read from the buffer.
Do you have an idea how to achieve this?