Dockerized Nvidia Cuda 11.8 Gradio App

docker-compose.yml

services:
  app:
    build: .
    ports:
        - "7860:7860"
    environment:
        GRADIO_SERVER_NAME: "0.0.0.0"    
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

Dockerfile

# Since we're using the nvidia/cuda base image, this requires nvidia-container-toolkit installed on the host system to pass through the drivers to the container.
# see: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
# If you've just installed nvidia-container-toolkit, you may need to restart the docker daemon before it will detect the GPU
FROM nvidia/cuda:12.3.0-runtime-ubuntu22.04 AS final
WORKDIR /app

# Install Git and Git LFS
RUN apt-get update && apt-get install -y curl wget
RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && apt-get install -y git-lfs

#need to install NVIDIA's gpg key before apt search will show up to date packages for cuda
RUN wget -N -t 5 -T 10 http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \
    && dpkg -i ./cuda-keyring_1.1-1_all.deb

# install CUDA dependencies required according to `ldd libonnxruntime_providers_cuda.so`
RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-11-8 libcudnn8=8.9.1.23-1+cuda11.8

# According to `ldd libortextensions.so` it depends on ssl 1.1 to run, and the dotnet/runtime-deps base image installs it which is why it works inside the dotnet base images.
# Since we need access to the GPU to use the CUDA execution provider we need to use the nvidia/cuda base image instead.
# The nvidia/cuda base image doesn't contain SSL 1.1, hence we have to manually install it like this ot satisfy the dependency.
# This fixes the "The ONNX Runtime extensions library was not found" error.
# See: https://stackoverflow.com/questions/72133316/libssl-so-1-1-cannot-open-shared-object-file-no-such-file-or-directory
# If you get 404 not found go to http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/ and find the new version and update the code below, 
RUN wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.23_amd64.deb && dpkg -i libssl1.1_1.1.1f-1ubuntu2.23_amd64.deb

# installing requirements
RUN apt-get update 
RUN apt-get install ffmpeg -y
RUN apt-get install python3.10 python3-pip python3-dev -y
RUN pip install --upgrade pip 

COPY requirements.txt .

RUN pip install -r ./requirements.txt 

COPY . .

ENTRYPOINT ["sh", "-c", "nvidia-smi && python3 run.py"]

run.sh

#! /bin/bash

docker compose up --build