Naive example of GCS memory consumption when writing many small objects

main.go

package main

import (
	"context"
	"crypto/rand"
	"log"
	"os"
	"time"

	"cloud.google.com/go/storage"
	"github.com/google/uuid"

	"net/http"
	_ "net/http/pprof"
)

func main() {
	go func() {
		http.ListenAndServe(":6060", nil)
	}()
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Fatal("error creating gcs client: ", err)
	}
	bucket := client.Bucket(os.Getenv("GCS_BUCKET"))

	for i := 0; i < 100; i++ {
		go func() {
			value := make([]byte, 834)
			rand.Read(value)

			wr := bucket.Object(uuid.New().String()).NewWriter(ctx)

			if _, err = wr.Write(value); err != nil {
				log.Println("error writing: ", err)
			}

			if err = wr.Close(); err != nil {
				log.Println("error closing: ", err)
			}
		}()
	}

	time.Sleep(60 * time.Minute)
}