AWS S3 upload using signature v4

s3-upload-aws4.sh

#!/bin/sh

# To the extent possible under law, Viktor Szakats
# has waived all copyright and related or neighboring rights to this
# script.
# CC0 - https://creativecommons.org/publicdomain/zero/1.0/
# SPDX-License-Identifier: CC0-1.0

# THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# Upload a file to Amazon AWS S3 (and compatible) using Signature Version 4
#
# docs:
#   https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
#   https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
#
# requires:
#   curl, openssl 1.x or newer, GNU sed, LF EOLs in this file

# shellcheck disable=SC2317

# shellcheck disable=SC3040
set -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o pipefail

fileLocal="${1:-example-local-file.ext}"
bucket="${2:-example-bucket}"  # AWS S3 bucket or full URL (with ending slash): https://localhost[:9000]/[bucket/]
region="${3:-}"
storageClass="${4:-STANDARD}"  # or 'REDUCED_REDUNDANCY'
SSE="${5:-AES256}"; [ "${SSE}" = 'none' ] && SSE=''  # Server-side encryption: 'AES256' (default) or 'none'

my_openssl() {
  if [ -f /usr/local/opt/[email protected]/bin/openssl ]; then
    /usr/local/opt/[email protected]/bin/openssl "$@"
  elif [ -f /usr/local/opt/openssl/bin/openssl ]; then
    /usr/local/opt/openssl/bin/openssl "$@"
  else
    openssl "$@"
  fi
}

my_sed() {
  if command -v gsed > /dev/null 2>&1; then
    gsed "$@"
  else
    sed "$@"
  fi
}

awsStringSign4() {
  kSecret="AWS4$1"
  kDate=$(printf         '%s' "$2" | my_openssl dgst -sha256 -hex -mac HMAC -macopt "key:${kSecret}"     2>/dev/null | my_sed 's/^.* //')
  kRegion=$(printf       '%s' "$3" | my_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kDate}"    2>/dev/null | my_sed 's/^.* //')
  kService=$(printf      '%s' "$4" | my_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kRegion}"  2>/dev/null | my_sed 's/^.* //')
  kSigning=$(printf 'aws4_request' | my_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kService}" 2>/dev/null | my_sed 's/^.* //')
  signedString=$(printf  '%s' "$5" | my_openssl dgst -sha256 -hex -mac HMAC -macopt "hexkey:${kSigning}" 2>/dev/null | my_sed 's/^.* //')
  printf '%s' "${signedString}"
}

iniGet() {
  # based on: https://stackoverflow.com/questions/22550265/read-certain-key-from-certain-section-of-ini-file-sed-awk#comment34321563_22550640
  printf '%s' "$(my_sed \
    -n -E "/\[$2\]/,/\[.*\]/{/$3/s/(.*)=[ \\t]*(.*)/\2/p}" "$1")"
}

# Initialize access keys

if [ -z "${AWS_CONFIG_FILE:-}" ]; then
  if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then
    >&2 echo '! AWS_CONFIG_FILE or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY envvars not set.'
    exit 1
  else
    awsAccess="${AWS_ACCESS_KEY_ID}"
    awsSecret="${AWS_SECRET_ACCESS_KEY}"
    awsRegion="${AWS_DEFAULT_REGION:-eu-west-1}"
  fi
else
  awsProfile='default'

  # Read standard aws-cli configuration file
  # pointed to by the envvar AWS_CONFIG_FILE
  awsAccess="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'aws_access_key_id')"
  awsSecret="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'aws_secret_access_key')"
  awsRegion="$(iniGet "${AWS_CONFIG_FILE}" "${awsProfile}" 'region')"
fi

# Initialize defaults

if [ -z "${region}" ]; then
  region="${awsRegion}"
fi

>&2 echo "! Uploading..." "${fileLocal}" "->" "${bucket}" "${region}" "${storageClass}"
>&2 echo "! | $(uname) | $(my_openssl version) | $(my_sed --version | head -1) |"

# Initialize helper variables

httpReq='PUT'
authType='AWS4-HMAC-SHA256'
service='s3'
if [ "${bucket#https://*}" != "${bucket}" ] || \
   [ "${bucket#http://*}" != "${bucket}" ]; then
  fullUrl="${bucket}"
else
  fullUrl="https://${bucket}.${service}.${region}.amazonaws.com/"
fi
pathRemote="$(printf '%s' "${fullUrl}" | sed -E 's|^https?://||g' | grep -o -E '/.*$' | cut -c 2-)"
hostport="$(printf '%s' "${fullUrl}" | sed -E -e 's|^https?://||g' -e 's|/.*$||')"
dateValueS=$(date -u +'%Y%m%d')
dateValueL=$(date -u +'%Y%m%dT%H%M%SZ')
if command -v file >/dev/null 2>&1; then
  contentType="$(file --brief --mime-type "${fileLocal}")"
else
  contentType='application/octet-stream'
fi

# Try to URL-encode the filename we pass

# based on: https://gist.github.com/jaytaylor/5a90c49e0976aadfe0726a847ce58736?permalink_comment_id=4043195#gistcomment-4043195
# Very curl version dependent.
fileRemote="$({ curl --silent --get / --data-urlencode "=${fileLocal}" --write-out '%{url}' 2>/dev/null || true; } | cut -c 3- | sed 's/+/%20/g')"
if [ -z "${fileRemote}" ] || [ "${fileRemote}" = '/' ]; then
  # Needs trurl
  fileRemote="$({ trurl --accept-space "file:///${fileLocal}" 2>/dev/null || true; } | cut -c 9-)"
  if [ -z "${fileRemote}" ]; then
    # Needs python3
    fileRemote="$({ printf '%s' "${fileLocal}" | python3 \
      -c 'import sys; import urllib.parse as ul; sys.stdout.write(ul.quote_plus(sys.stdin.read()))' 2>/dev/null || true; } | sed 's/+/%20/g')"
    if [ -z "${fileRemote}" ]; then
      # Last resort, that will probably not work as expected, but better than an empty string
      fileRemote="${fileLocal}"
    fi
  fi
fi

# 0. Hash the file to be uploaded

if [ -f "${fileLocal}" ]; then
  payloadHash=$(my_openssl dgst -sha256 -hex < "${fileLocal}" 2>/dev/null | my_sed 's/^.* //')
else
  >&2 echo "! File not found: '${fileLocal}'"
  exit 1
fi

# 1. Create canonical request

# NOTE: order significant in ${headerList} and ${canonicalRequest}

if [ -n "${SSE}" ]; then
  headerList='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption;x-amz-storage-class'
  ssehdr="\
x-amz-server-side-encryption:${SSE}
"
else
  headerList='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class'
  ssehdr=''
fi

canonicalRequest="\
${httpReq}
/${pathRemote}${fileRemote}

content-type:${contentType}
host:${hostport}
x-amz-content-sha256:${payloadHash}
x-amz-date:${dateValueL}
${ssehdr}x-amz-storage-class:${storageClass}

${headerList}
${payloadHash}"

# Hash it

canonicalRequestHash=$(printf '%s' "${canonicalRequest}" | my_openssl dgst -sha256 -hex 2>/dev/null | my_sed 's/^.* //')

# 2. Create string to sign

stringToSign="\
${authType}
${dateValueL}
${dateValueS}/${region}/${service}/aws4_request
${canonicalRequestHash}"

# 3. Sign the string

signature=$(awsStringSign4 "${awsSecret}" "${dateValueS}" "${region}" "${service}" "${stringToSign}")

# Upload

curl --silent --location --proto-redir =https --request "${httpReq}" --upload-file "${fileLocal}" \
  --header "Content-Type: ${contentType}" \
  --header "Host: ${hostport}" \
  --header "X-Amz-Content-SHA256: ${payloadHash}" \
  --header "X-Amz-Date: ${dateValueL}" \
  --header "X-Amz-Server-Side-Encryption: ${SSE}" \
  --header "X-Amz-Storage-Class: ${storageClass}" \
  --header "Authorization: ${authType} Credential=${awsAccess}/${dateValueS}/${region}/${service}/aws4_request, SignedHeaders=${headerList}, Signature=${signature}" \
  "${fullUrl}${fileRemote}"

return

# Examples

cat > 'test.xml' <<EOF
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test>
</test>
EOF

export AWS_ACCESS_KEY_ID='<example-id>'
export AWS_SECRET_ACCESS_KEY='<example-key>'

./s3-upload-aws4.sh 'test.xml' 'http://localhost:9000/example-bucket/' 'eu-west-1' '' 'none'
./s3-upload-aws4.sh 'test.xml' 'http://localhost:9000/example-bucket/' 'eu-west-1' ''
./s3-upload-aws4.sh 'test.xml' 'example-bucket' 'eu-west-1' ''

That's correct. There have been fixes for this in later curl versions up to recently (v8.5.0), so the newer the curl, the better this works.

its weird how it works for me on linux but the same script does not work on mac
Signature error