Measure maximum VRAM consumption of a program on linux for Nvidia devices

gpumeter.sh

#!/bin/bash

if [ "$#" -lt 1 ]; then
    echo "Usage: $0 executable [args...]"
    exit 1
fi

# Start monitoring GPU memory usage in the background
nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -l 1 > gpu_usage.log &
PID=$!

# Execute the provided script
eval $@

# Kill the nvidia-smi monitoring process
kill $PID

# Calculate and display the max GPU memory usage
MAX_USAGE=$(awk '{ if(max < $1) { max=$1 } } END { print max }' gpu_usage.log)
echo "Maximum GPU VRAM usage: ${MAX_USAGE}MiB"