FROM nvidia/cuda:11.3.1-base-ubuntu20.04 AS base_image ENV DEBIAN_FRONTEND=noninteractive \ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" RUN apt-get update \ && apt-get upgrade -y \ && apt-get autoremove -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* FROM base_image AS ec2 LABEL maintainer="Amazon AI" LABEL dlc_major_version="1" # Specify accept-bind-to-port LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true ARG MMS_VERSION=1.1.8 ARG PYTHON=python3 ARG PYTHON_VERSION=3.8.10 ENV CUDNN_VERSION=8.2.0.53 ENV NCCL_VERSION=2.10.3 ENV EFA_VERSION=1.11.2 ENV OMPI_VERSION=4.1.1 ENV NVML_VERSION=11.3.58 ARG OPEN_MPI_VERSION=4.0.1 ARG CUBLAS_VERSION=11.5.1.109 ARG MAMBA_VERSION=4.12.0-0 ENV TORCH_CUDA_ARCH_LIST="3.7 5.0 7.0+PTX 8.0" # Set MKL_THREADING_LAYER=GNU to prevent issues between torch and numpy/mkl ENV MKL_THREADING_LAYER=GNU ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" ENV LD_LIBRARY_PATH="/opt/conda/lib:${LD_LIBRARY_PATH}" # HF ARGS ARG PT_INFERENCE_URL=https://pytorch-ei-binaries.s3.us-west-2.amazonaws.com/r1.10.2_inference/20220211-013556/528b6ca3747ec951396383f2a70d1354283ad29f/gpu/torch-1.10.2%2Bcu113-cp38-cp38-manylinux1_x86_64.whl ARG PT_TORCHAUDIO_URL=https://framework-binaries.s3.us-west-2.amazonaws.com/pytorch/1.10.2/torchaudio-0.10.2%2Bcu113-cp38-cp38-linux_x86_64.whl ARG TRANSFORMERS_VERSION ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PYTHONIOENCODING=UTF-8 \ LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ TEMP=/home/model-server/tmp \ DEBIAN_FRONTEND=noninteractive ENV PATH /opt/conda/bin:$PATH RUN apt-get update \ # TODO: Remove systemd upgrade once it is updated in base image && apt-get -y upgrade --only-upgrade systemd openssl cryptsetup \ && apt-get install -y --no-install-recommends software-properties-common \ && add-apt-repository ppa:openjdk-r/ppa \ && apt-get update \ && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \ build-essential \ ca-certificates \ cmake \ cuda-command-line-tools-11-3 \ cuda-cudart-11-3 \ cuda-libraries-dev-11-3 \ curl \ emacs \ git \ jq \ libcublas-11-3=${CUBLAS_VERSION}-1 \ libcublas-dev-11-3=${CUBLAS_VERSION}-1 \ libcudnn8=$CUDNN_VERSION-1+cuda11.3 \ libcufft-dev-11-3 \ libcurand-dev-11-3 \ libcusolver-dev-11-3 \ libcusparse-dev-11-3 \ cuda-nvml-dev-11-3=${NVML_VERSION}-1 \ libgl1-mesa-glx \ libglib2.0-0 \ libgomp1 \ libibverbs-dev \ libnuma1 \ libnuma-dev \ libsm6 \ libssl1.1 \ libxext6 \ libxrender-dev \ openjdk-8-jdk-headless \ openssl \ vim \ wget \ unzip \ zlib1g-dev \ libsndfile1-dev \ ffmpeg \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean RUN cd /tmp \ && git clone https://github.com/NVIDIA/nccl.git -b v${NCCL_VERSION}-1 \ && cd nccl \ && make -j64 src.build BUILDDIR=/usr/local \ && rm -rf /tmp/nccl RUN curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-x86_64.sh \ && chmod +x ~/mambaforge.sh \ && ~/mambaforge.sh -b -p /opt/conda \ && rm ~/mambaforge.sh \ && /opt/conda/bin/conda install -c conda-forge \ python=$PYTHON_VERSION \ cython \ "mkl-include==2021.4.0" \ "mkl==2021.4.0" \ botocore \ # Below 2 are included in miniconda base, but not mamba so need to install conda-content-trust \ charset-normalizer \ && /opt/conda/bin/conda clean -ya # Upstream conda looks to have moved to 4.13 which is incompatible with mamba 0.22.1 and will fail the conda-forge installs. # having "conda update conda" before the "conda -c conda-forge" commands will automatically update conda to 4.13. # Moving conda update conda" after the "conda -c conda-forge" commands keep conda at 4.12 but will update other packages using # the current conda 4.12 RUN /opt/conda/bin/conda update -y conda \ && /opt/conda/bin/conda clean -ya RUN pip install --upgrade pip --trusted-host pypi.org --trusted-host files.pythonhosted.org \ && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 \ && pip install packaging==20.4 \ enum-compat==0.0.3 \ "cryptography>3.2" # Conda installs links for libtinfo.so.6 and libtinfo.so.6.2 both # Which causes "/opt/conda/lib/libtinfo.so.6: no version information available" warning # Removing link for libtinfo.so.6. This change is needed only for ubuntu 20.04-conda, and can be reverted # once conda fixes the issue: https://github.com/conda/conda/issues/9680 RUN rm -rf /opt/conda/lib/libtinfo.so.6 RUN wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-$OPEN_MPI_VERSION.tar.gz \ && gunzip -c openmpi-$OPEN_MPI_VERSION.tar.gz | tar xf - \ && cd openmpi-$OPEN_MPI_VERSION \ && ./configure --prefix=/home/.openmpi \ && make all install \ && cd .. \ && rm openmpi-$OPEN_MPI_VERSION.tar.gz \ && rm -rf openmpi-$OPEN_MPI_VERSION ENV PATH="$PATH:/home/.openmpi/bin" ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/.openmpi/lib/" WORKDIR / RUN pip install --no-cache-dir \ multi-model-server==$MMS_VERSION \ sagemaker-inference RUN cd tmp/ \ && rm -rf tmp* RUN useradd -m model-server \ && mkdir -p /home/model-server/tmp \ && chown -R model-server /home/model-server COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py COPY config.properties /etc/sagemaker-mms.properties RUN chmod +x /usr/local/bin/dockerd-entrypoint.py ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py ################################# # Hugging Face specific section # ################################# RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.10/license.txt -o /license.txt # Uninstall and re-install torch and torchvision from the PyTorch website RUN pip uninstall -y torch \ && pip install --no-cache-dir -U $PT_INFERENCE_URL $PT_TORCHAUDIO_URL # install Hugging Face libraries and its dependencies RUN pip install --no-cache-dir \ transformers[sentencepiece,audio,vision]==${TRANSFORMERS_VERSION} \ "protobuf>=3.19.5,<3.20" \ numpy>=1.21.5 \ "sagemaker-huggingface-inference-toolkit<3" RUN HOME_DIR=/root \ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* EXPOSE 8080 8081 ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] CMD ["serve"]