CloudInitScript.sh

#!/bin/bash
set -x #echo on

## Parameters
#LOCAL_RUN=1
if [ "$LOCAL_RUN" == "1" ]; then
    # not necessary for a local run:
    EGORBOT_HOST=""

    # Configurable:
    GH_PR_ID="106525"
    GH_IS_PR="1"
    GH_PR_BENCH_LINK="https://gist.githubusercontent.com/EgorBo/ba2938d76bebccf76f547b423bcc9a21/raw"
    GH_PH_BENCH_ARGS="";
    BENCH_TYPE="3" # has no Main() and no top-level-statement

    # Defaults
    EGORBOT_JOBID="111111"
    JOB_RUN_PERF="0"
    JOB_RUN_NODELAY_PERF="0"
    USE_MONOVM="0"
    VM_CPU="intel"
    PERFLAB_ENABLED="0"
    GH_BASE=""
    GH_DIFF=""
    PERF_EVENT=""
    NO_NATIVE_PGO="0"
fi

OS=$(uname)
UNAMEA=$(uname -m)
# Check if it matches 'arm64' or 'aarch64'
if [[ "$UNAMEA" == "arm64" ]] || [[ "$UNAMEA" == "aarch64" ]]; then
    ARCH=arm64
else
    ARCH=x64
fi

DEFAULT_TFM="net9.0"
DEFAULT_OS="linux"

# Some global environment variables:

# Improve consistency of measurements
export DOTNET_JitEnableOptionalRelocs=0
export DOTNET_EnableWriteXorExecute=0 # not sure this affects the consistency.
# TODO: make loop alignment more aggressive or even disable it ?
# JitAlignLoopAdaptive=0

DIR_ROOT=/home
if [ "$OS" == "Darwin" ]; then
    DIR_ROOT=~/egorbot
    DEFAULT_OS="osx"
    # no perf on mac:
    JOB_RUN_PERF=0
    JOB_RUN_NODELAY_PERF=0
fi
DIR_WORK=$DIR_ROOT/egorbot
DIR_BENCHAPP=$DIR_WORK/benchapp

# if JitDisasm is set, users will be able to find the Jit StdOut in the zipped file
export DOTNET_JitStdOutFile=${DIR_ROOT}/egorbot/benchapp/BenchmarkDotNet.Artifacts/JitStdOut.asm

# We rarely are interested in DOTNET_JitDisasm of non-optimized code, users can opt-in it via --envvars
export DOTNET_JitDisasmOnlyOptimized=1

# Render 32b boundaries
export DOTNET_JitDisasmWithAlignmentBoundaries=1

# basic script dependecies:
if [ "$OS" != "Darwin" ]; then
    apt-get install -y jq
    apt-get install -y zip
else
    brew install jq
fi

################################################################################
# Send the result of the job to the host and exit
################################################################################
# $1 - "true" or "false" (success or failure)
# $2 - error message
function sendResultToAgent () {
    HOST_SUCCESS=$1
    HOST_ERROR=$2
    sleep 3
    cd $DIR_WORK

    zip ${DIR_WORK}/agent_logs.zip ${DIR_ROOT}/agent.log

    CURL_CMD="curl -k -X POST \"http://${EGORBOT_HOST}/StopJob?jobId=${EGORBOT_JOBID}&success=${HOST_SUCCESS}\" \
     -F \"file=@${DIR_WORK}/agent_logs.zip\""
    if [[ -f "${DIR_WORK}/BDN_Artifacts.zip" ]]; then
        CURL_CMD+=" -F \"file=@${DIR_WORK}/BDN_Artifacts.zip\""
    fi

    eval $CURL_CMD

    if [ "$HOST_SUCCESS" == "false" ]; then
        exit 1
    else
        exit 0
    fi
}

################################################################################
# Run the benchmark with perf
################################################################################
# $1 - perf run name ("base" or "diff")
function RunPerf () {
    export DOTNET_JitStdOutFile=""
    
    PERF_RECORD_ARGS=""
    # cycles/cpu-cycles are not available on arm64 VMs :'( so we use cpu-clock instead
    # if [[ "$ARCH" == "arm64" ]]; then
    PERF_RECORD_ARGS="-e cpu-clock"
    # fi

    # NOTE: should we avoid doing publish + inprocesstolchain ?
    cd $DIR_BENCHAPP

    TFM_PATH=$DIR_WORK/benchapp/bin/Release/${DEFAULT_TFM}
    
    # Run the benchmark and then attach with perf to it (to skip the warmup phase)

    if [ "$JOB_RUN_NODELAY_PERF" == "0" ]; then

        dotnet publish -c Release -f ${DEFAULT_TFM} --sc
        pkill corerun || true
        pkill dotnet || true

        # zero out CUSTOM_CORERUN if no local runtime is used
        if [ "$LOCAL_RUNTIME" == "0" ]; then
            CUSTOM_CORERUN=""
            # TODO: we might need to run dotnet-symbol in this case?
        fi
        
        PERF_STAT=""

        # if PERF_EVENT is set, we use it instead of the default one
        if [ ! -z "$PERF_EVENT" ]; then
            PERF_STAT="-e $PERF_EVENT"
            PERF_RECORD_ARGS="-e $PERF_EVENT"
        else
            # AWS Gravitons
            if [ "$VM_CPU" == "Arm64" ]; then
                PERF_STAT="-e cpu-cycles,context-switches,page-faults,dTLB-loads,dTLB-load-misses,iTLB-loads,iTLB-load-misses,ld_align_lat,st_align_lat,inst_retired,l1d_cache_lmiss_rd,br_mis_pred_retired,br_retired,l2d_cache_lmiss_rd,l3d_cache_lmiss_rd,ll_cache_miss_rd,dmb_spec,dsb_spec,isb_spec,unaligned_ld_spec,unaligned_st_spec"
            fi
        fi

        MBENCHES_CNT=$(dotnet $TFM_PATH/${DEFAULT_OS}-$ARCH/publish/benchapp.dll --filter "*" --list flat | wc -l)

        echo "FFound $MBENCHES_CNT benchmarks"
        echo "$(dotnet $TFM_PATH/${DEFAULT_OS}-$ARCH/publish/benchapp.dll --filter "*" --list flat)"

        if [ "$MBENCHES_CNT" -gt "3" ]; then
            echo "Too many benchmarks ($MBENCH_COUNT > 3)!\n"
            sleep 5
            sendResultToAgent "false" "Too many [Benchmark]s to profile with perf, not more than 3 please."
        fi

        for bdnline in $(dotnet $TFM_PATH/${DEFAULT_OS}-$ARCH/publish/benchapp.dll --filter "*" --list flat); do
            sleep 3
            pkill -9 -f dotnet || true
            pkill -9 -f corerun || true

            # loop over MBENCH_COUNT [0..$MBENCH_COUNT)
            # Run the BDN benchmark without perf and ask it to spin the first [Benchmark], basically, forever
            DOTNET_JitEnableOptionalRelocs=0 \
            DOTNET_JitStdOutFile="" \
            DOTNET_PerfMapShowOptimizationTiers=1 \
            DOTNET_JitFramed=1 \
            DOTNET_PerfMapEnabled=1 \
            DOTNET_EnableWriteXorExecute=0 \
            $DIR_WORK/core_root_$1/corerun $TFM_PATH/${DEFAULT_OS}-$ARCH/publish/benchapp.dll --filter "$bdnline" -i \
            --noForcedGCs --noOverheadEvaluation --disableLogFile --maxWarmupCount 8 --minIterationCount 15000000 --maxIterationCount 20000000 -a perfarts &

            # Escape characters that are not allowed in file names for $bdnline
            bdnline_escaped=$(echo $bdnline | sed 's/[^a-zA-Z0-9]/_/g')

            CURR_BENCH=$DIR_BENCHAPP/BenchmarkDotNet.Artifacts/PerfBench__${bdnline_escaped}
            mkdir $CURR_BENCH
            
            # Wait for 40 seconds to skip the warmup phase and attach to the process with 'perf record'
            sleep 40
            perf record $PERF_RECORD_ARGS -k 1 -g -F 1999 -p $(pgrep corerun) sleep 5
            sleep 2
            perf record $PERF_RECORD_ARGS -k 1 -g -F 199 -p $(pgrep corerun) -o perf_small.data sleep 5
            sleep 2
            perf stat $PERF_STAT -o $CURR_BENCH/$1.stats -p $(pgrep corerun) sleep 6
            perf list > $CURR_BENCH/$1.perf_list.txt
            pkill -9 -f dotnet || true
            pkill -9 -f corerun || true

            # Now symbolize the perf.data file
            perf inject --input perf.data --jit --output perfjit.data
            perf inject --input perf_small.data --jit --output perfjit_small.data

            perf report --input perfjit.data --no-children --percent-limit 2 --stdio > $CURR_BENCH/$1_functions.txt
            perf annotate --stdio2 -i perfjit.data --percent-limit 2 -M intel > $CURR_BENCH/$1.asm

            # For speedscope:
            perf script -i perfjit_small.data > $CURR_BENCH/speedscope_$1_${EGORBOT_JOBID}.txt

            # Generate flamegraph (svg)
            cd $DIR_BENCHAPP/FlameGraph
            cp ../perfjit.data ./perf.data
            perf script | ./stackcollapse-perf.pl |./flamegraph.pl > $CURR_BENCH/$1_flamegraph.svg
            cd $DIR_BENCHAPP
        done

    else
        # Rare case: we want to run the benchmark (actually, it's not a BDN benchmark at all) without any delay
        # to measure the startup phase
        dotnet publish -c Release -f ${DEFAULT_TFM} --sc /p:AllowUnsafeBlocks=True /p:EnablePreviewFeatures=True
        pkill corerun || true
        pkill dotnet || true

        DOTNET_JitEnableOptionalRelocs=0 \
        DOTNET_JitStdOutFile="" \
        DOTNET_PerfMapShowOptimizationTiers=1 \
        DOTNET_JitFramed=1 \
        DOTNET_PerfMapEnabled=1 \
        DOTNET_EnableWriteXorExecute=0 \
        perf record $PERF_RECORD_ARGS -k 1 -g -F 49999 $DIR_WORK/core_root_$1/corerun $TFM_PATH/${DEFAULT_OS}-$ARCH/publish/benchapp.dll

        # Now symbolize the perf.data file
        sleep 5
        perf inject --input perf.data --jit --output perfjit.data
        perf report --input perfjit.data --no-children --percent-limit 2 --stdio > $DIR_BENCHAPP/BenchmarkDotNet.Artifacts/$1_functions.txt
        perf annotate --stdio2 -i perfjit.data --percent-limit 2 -M intel > $DIR_BENCHAPP/BenchmarkDotNet.Artifacts/$1.asm
        
        cd $DIR_BENCHAPP/FlameGraph
        cp ../perfjit.data ./perf.data
        perf script | ./stackcollapse-perf.pl |./flamegraph.pl > $DIR_BENCHAPP/BenchmarkDotNet.Artifacts/$1_flamegraph.svg
    fi
}

################################################################################
# Emit a section header to the log
################################################################################
# $1 - section name
function startSection () {
    echo -e "\n\n#################################################################"
    echo "# $1"
    echo -e "#################################################################\n"
}

################################################################################
# Clone the runtime and build it (for baseline)
################################################################################
function cloneAndBuildBase () {
    # Clone dotnet/runtime
    cd $DIR_WORK
    git clone --no-tags --single-branch --quiet https://github.com/dotnet/runtime.git
    cd runtime
    git log -1
    git config --global user.email [email protected]
    git config --global user.name egorbot
    eng/./install-native-dependencies.sh

    # do on ${DEFAULT_OS}:
    if [ "$OS" != "Darwin" ]; then
        apt install -y zlib1g-dev # is this still needed?
    fi

    BUILD_SUBSET="Clr+Libs"
    if [ "$USE_MONOVM" == "1" ]; then
        BUILD_SUBSET="Mono+Libs"
    fi

    BUILD_PROPS=" /p:RunAnalyzers=false /p:ApiCompatValidateAssemblies=false "
    if [ "$NO_NATIVE_PGO" == "1" ]; then
        BUILD_PROPS="${BUILD_PROPS} /p:NoPgoOptimize=true "
    fi

    # revert to GH_BASE commit if it's set (not null or empty):
    if [ -n "$GH_BASE" ]; then
        echo "GH_BASE is set to $GH_BASE, reverting to it"
        git checkout $GH_BASE
    fi
 
    startSection "Building BASE runtime"

    if ./build.sh $BUILD_SUBSET -c Release  ${BUILD_PROPS} ; then
        echo "Base runtime build successful"
    else
        sendResultToAgent "false" "Base runtime build failed"
    fi

    if [ "$USE_MONOVM" == "1" ]; then
        ./build.sh Clr -c Release ${BUILD_PROPS}
        cp artifacts/bin/mono/${DEFAULT_OS}.$ARCH.Release/* artifacts/bin/coreclr/${DEFAULT_OS}.$ARCH.Release/
    fi
    
    src/tests/./build.sh Release generatelayoutonly
    cp -rf artifacts/tests/coreclr/${DEFAULT_OS}.$ARCH.Release/Tests/Core_Root $DIR_WORK/core_root_base
}

################################################################################
# Rebuild the runtime (for diff)
################################################################################
function buildDiffRuntime () {

    BUILD_PROPS=" /p:RunAnalyzers=false /p:ApiCompatValidateAssemblies=false "
    if [ "$NO_NATIVE_PGO" == "1" ]; then
        BUILD_PROPS="${BUILD_PROPS} /p:NoPgoOptimize=true "
    fi

    if ./build.sh $BUILD_SUBSET -c Release ${BUILD_PROPS} ; then
        echo "Diff runtime build successful"
    else
        sendResultToAgent "false" "Diff runtime build failed"
    fi

    if [ "$USE_MONOVM" == "1" ]; then
        cp artifacts/bin/mono/${DEFAULT_OS}.$ARCH.Release/* artifacts/bin/coreclr/${DEFAULT_OS}.$ARCH.Release/
    fi

    src/tests/./build.sh Release generatelayoutonly
    cp -rf artifacts/tests/coreclr/${DEFAULT_OS}.$ARCH.Release/Tests/Core_Root $DIR_WORK/core_root_diff
}

################################################################################
# Build the benchmark app and inspect the benchmarks via --list
################################################################################
function buildBenchmark () {
    # preprare the benchmark (TODO: validate it earlier)  
    cd $DIR_BENCHAPP
    dotnet new console -f ${DEFAULT_TFM}
    wget -O Program.cs "$GH_PR_BENCH_LINK"
    wget -O benchapp.csproj https://gist.githubusercontent.com/EgorBo/c3378873ad204ebf522a07138f621128/raw

    # if BENCH_TYPE is BenchmarkClass then we need to add an entrypoint file:
    if [ "$BENCH_TYPE" == "3" ]; then
        echo "BenchmarkDotNet.Running.BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);" > Entrypoint.cs
    fi

    dotnet add package BenchmarkDotNet
    dotnet add package System.IO.Hashing --prerelease

    if dotnet build -c Release -f ${DEFAULT_TFM} -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True ; then
        echo "Benchmark build successful"
    else
        sendResultToAgent "false" "Benchmark failed to build"
    fi
}

################################################################################
# Install a specific version of dotnet locally
################################################################################
# $1 - dotnet version, e.g. "7.0"
function installDotnet () {
    # download dotnet-install.sh script if it's not already downloaded
    if [ ! -f "${DIR_ROOT}/dotnet-install.sh" ]; then
        wget https://dot.net/v1/dotnet-install.sh -O ${DIR_ROOT}/dotnet-install.sh
        chmod +x ${DIR_ROOT}/dotnet-install.sh
    fi
    ${DIR_ROOT}/./dotnet-install.sh -Channel $1 -InstallDir ${DIR_ROOT}/dotnet
}

################################################################################
# Prepare and run PerfLab (dotnet/performance) benchmarks
################################################################################
function preparePerfLabBenchmarks () {
    cd $DIR_WORK
    git clone --no-tags --single-branch --depth 1 --quiet https://github.com/dotnet/performance

    cd $DIR_WORK/performance/src/benchmarks/micro
    if dotnet build -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True ; then
        echo "ok"
    else
        sendResultToAgent "false" "Perflab microbenchmarks failed to build"
    fi
    
    BENCH_COUNT=$(dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --list flat | wc -l)

    # Check if the filter returned too many benchmarks (we don't want to run too many)
    if [ "$BENCH_COUNT" -gt "20" ]; then
        echo "Too many benchmarks ($BENCH_COUNT > 20)! BDN --list flat printed:\n"
        dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --list flat
        sleep 5
        sendResultToAgent "false" "filter returned too many benchmarks ($BENCH_COUNT > 20)"
    fi
    
    # Check if the filter returned 0 benchmarks
    if [ "$BENCH_COUNT" == "0" ]; then
        echo "0 benchmarks found! BDN --list flat printed:\n"
        dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --list flat
        sleep 5
        sendResultToAgent "false" "filter returned 0 benchmarks"
    fi
}

################################################################################
# Run PerfLab (dotnet/performance) benchmarks
################################################################################
function runPerfLabBenchmarks () {
    cd $DIR_WORK/performance/src/benchmarks/micro

    if dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --corerun $BASE_CORERUN $DIFF_CORERUN -h Job StdDev RatioSD Median Min Max ; then
        echo "PerfLab Benchmark run successful (JobId=${EGORBOT_JOBID})"
    else
        sendResultToAgent "false" "PerfLab Benchmark run failed"
    fi

    cp -r $DIR_WORK/performance/artifacts/bin/MicroBenchmarks/Release/${DEFAULT_TFM}/BenchmarkDotNet.Artifacts $DIR_BENCHAPP/
}

# Create a ramdisk for faster builds (I wasn't able to notice any difference, but Miha did)
#mkdir /ramdisk
#mount -t tmpfs -o size=40G tmpfs /ramdisk
#DIR_ROOT=/ramdisk

echo "Starting job for PR ${GH_PR_ID} (JobId=${EGORBOT_JOBID})"

mkdir $DIR_WORK
mkdir $DIR_BENCHAPP
mkdir $DIR_BENCHAPP/BenchmarkDotNet.Artifacts

# install bootstrap dotnets
installDotnet "9.0"
installDotnet "8.0"
DOTNET_ROOT=${DIR_ROOT}/dotnet
PATH=$DOTNET_ROOT:$DOTNET_ROOT/tools:$PATH

BENCH_ARGS=$(echo "$GH_PH_BENCH_ARGS" | base64 --decode)
echo "BENCH_ARGS: $BENCH_ARGS"

cd $DIR_WORK

if [ "$PERFLAB_ENABLED" == "1" ]; then
    startSection "Building PerfLab benchmarks"
    preparePerfLabBenchmarks
else
    startSection "Building benchmark app"
    buildBenchmark
fi

BASE_ONLY=0
BASE_CORERUN=$DIR_WORK/core_root_base/corerun
DIFF_CORERUN=

LOCAL_RUNTIME=0
# if is PR or base commit is set, we need to build the runtime
if [ "$GH_IS_PR" == "1" ] || [ -n "$GH_BASE" ]; then
    LOCAL_RUNTIME=1
fi

if [ "$LOCAL_RUNTIME" == "1" ]; then
    startSection "Cloning runtime and installing dependencies"
    cloneAndBuildBase

    startSection "Applying PR patch"
    cd $DIR_WORK/runtime
    if [ -n "$GH_DIFF" ]; then
        echo "GH_DIFF is set to $GH_DIFF, checking out"
        if [ "$GH_DIFF" == "previous" ]; then
            git reset --hard HEAD~1
        else
            git checkout $GH_DIFF
        fi
    elif [ -n "$GH_BASE" ]; then
        echo "GH_BASE is set while GH_DIFF is not - do nothing for diff."
        git checkout $GH_BASE
        BASE_ONLY=1
    elif [ "$GH_IS_PR" == "1" ]; then
        echo "Switching to PR branch"
        git fetch origin pull/$GH_PR_ID/head:PR_BRANCH
        git switch PR_BRANCH
    else
        echo "No PR or base/diff commit specified - run on main only"
        BASE_ONLY=1
    fi

    if [ "$BASE_ONLY" == "0" ]; then
        startSection "Building DIFF runtime"
        buildDiffRuntime
        DIFF_CORERUN=$DIR_WORK/core_root_diff/corerun
    fi
else
    JOB_RUN_PERF=0
    # Install additional dotnet versions (mainly for --runtimes ${DEFAULT_TFM} net8.0 net7.0 net6.0 to work)
    installDotnet "7.0"
    installDotnet "6.0"
    #installDotnet "5.0"
fi

cd $DIR_BENCHAPP

startSection "Running the benchmark app"

if [ "$LOCAL_RUNTIME" == "0" ]; then
    # needed for NAOT
    # apt install -y clang zlib1g-dev

    if [ "$PERFLAB_ENABLED" == "1" ]; then
        sendResultToAgent "false" "PerfLab is only supported for -commit flag"
    fi

    if dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --filter "*" -h StdDev RatioSD Median ; then
        echo "Benchmark run successful (JobId=${EGORBOT_JOBID})"
    else
        sendResultToAgent "false" "Benchmark run failed"
    fi
else
    if [ "$PERFLAB_ENABLED" == "1" ]; then
        # not supported yet for perflab benchmarks
        JOB_RUN_PERF=0
        runPerfLabBenchmarks
    else
        if dotnet run -c Release -f ${DEFAULT_TFM} -p:Nullable=disable -p:AllowUnsafeBlocks=True -p:EnablePreviewFeatures=True -- $BENCH_ARGS --filter "*" --corerun $BASE_CORERUN $DIFF_CORERUN -h Job StdDev RatioSD Median ; then
            echo "Benchmark run successful (JobId=${EGORBOT_JOBID})"
        else
            sendResultToAgent "false" "Benchmark run failed"
        fi
    fi
fi

if [ "$JOB_RUN_PERF" == "1" ]
then
    startSection "Running perf"
    cd $DIR_BENCHAPP

    # clone FlameGraph repo for generating flamegraphs
    git clone --depth 1 --quiet https://github.com/brendangregg/FlameGraph

    RunPerf "base"

    # skip diff if BASE_ONLY is set or LOCAL_RUNTIME is 0    
    if [ "$BASE_ONLY" == "0" ] && [ "$LOCAL_RUNTIME" == "1" ]; then
        RunPerf "diff"
    fi
fi

cd $DIR_BENCHAPP

startSection "Publishing results"

cd $DIR_WORK

ISPRBOOL="false"
if [ "$GH_IS_PR" == "1" ]; then
    ISPRBOOL="true"
fi

cd $DIR_BENCHAPP/BenchmarkDotNet.Artifacts
zip -r ${DIR_WORK}/BDN_Artifacts.zip .

echo "Job finished successfully"
sendResultToAgent "true" ""