FROM ubuntu:18.04 LABEL maintainer="Amazon AI" LABEL dlc_major_version="1" ARG PYTHON=python3 # Add arguments to achieve the version, python and url ARG PYTHON_VERSION=3.6.13 ARG OPEN_MPI_VERSION=4.0.1 # The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating # the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. ARG SMDEBUG_VERSION=1.0.9 # disable prompts during package installation ARG DEBIAN_FRONTEND=noninteractive # Python won’t try to write .pyc or .pyo files on the import of source modules # Force stdin, stdout and stderr to be totally unbuffered. Good for logging ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/conda/lib" ENV PATH=/opt/conda/bin:$PATH ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main ENV DGLBACKEND=pytorch ARG PT_TRAINING_URL=https://aws-pytorch-binaries.s3-us-west-2.amazonaws.com/r1.8.1_aws/20210325-012734/e1343088f0beb99438343e1e99e8d71ffb972b47/cpu/torch-1.8.1-cp36-cp36m-manylinux1_x86_64.whl ARG PT_TORCHVISION_URL=https://torchvision-build.s3-us-west-2.amazonaws.com/1.8.1/cpu/torchvision-0.9.1-cp36-cp36m-linux_x86_64.whl ARG PT_S3_WHL=https://aws-s3-plugin.s3-us-west-2.amazonaws.com/binaries/0.0.1/93fdaed/awsio-0.0.1-cp36-cp36m-manylinux1_x86_64.whl WORKDIR / RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ cmake \ curl \ emacs \ git \ jq \ libglib2.0-0 \ libgl1-mesa-glx \ libsm6 \ libxext6 \ libxrender-dev \ software-properties-common \ wget \ unzip \ vim \ zlib1g-dev \ && apt-get install -y -qq libkrb5-dev \ && apt-get install -y -qq libsasl2-dev libsasl2-modules \ && apt-get install -y -qq krb5-user \ && rm -rf /var/lib/apt/lists/* RUN wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-$OPEN_MPI_VERSION.tar.gz \ && gunzip -c openmpi-$OPEN_MPI_VERSION.tar.gz | tar xf - \ && cd openmpi-$OPEN_MPI_VERSION \ && ./configure --prefix=/home/.openmpi \ && make all install \ && cd .. \ && rm openmpi-$OPEN_MPI_VERSION.tar.gz \ && rm -rf openmpi-$OPEN_MPI_VERSION # The ENV variables declared below are changed in the previous section # Grouping these ENV variables in the first section causes # ompi_info to fail. This is only observed in CPU containers ENV PATH="$PATH:/home/.openmpi/bin" ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/.openmpi/lib/" RUN ompi_info --parsable --all | grep mpi_built_with_cuda_support:value # Copy workaround script for incorrect hostname COPY changehostname.c / COPY start_with_right_hostname.sh /usr/local/bin/start_with_right_hostname.sh # py37 is the oldest python version supported by Miniconda 4.10.3, py36 is installed below and will override py37 environment RUN curl -L -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh \ && chmod +x ~/miniconda.sh \ && ~/miniconda.sh -b -p /opt/conda \ && rm ~/miniconda.sh \ && /opt/conda/bin/conda install -c conda-forge \ python=$PYTHON_VERSION \ && /opt/conda/bin/conda install -y -c anaconda \ # conda 4.10.0 requires ruamel_yaml to be installed. Currently pinned at latest. ruamel_yaml==0.15.100 \ numpy \ ipython \ mkl \ mkl-include \ cython \ typing \ "pyopenssl>=17.5.0" \ && conda install -c conda-forge \ opencv \ && conda install -y \ scikit-learn \ pandas \ h5py \ requests \ && conda install -c dglteam -y dgl==0.6.1 \ && /opt/conda/bin/conda clean -ya \ && conda clean -ya # The following section uninstalls torch and torchvision before installing the # custom versions from an S3 bucket. This will need to be removed in the future RUN pip install --upgrade pip --trusted-host pypi.org --trusted-host files.pythonhosted.org \ && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 \ && pip install --no-cache-dir -U \ "pyyaml>=5.4,<5.5" \ "awscli<2" \ fastai==1.0.61 \ scipy==1.2.2 \ click \ "cryptography>3.2" \ smdebug==${SMDEBUG_VERSION} \ smclarify \ "sagemaker>=2,<3" \ sagemaker-experiments==0.* \ # Pin SM Training toolkit as the latest version (4.0.0) dropped py36 support "sagemaker-training<4" \ "sagemaker-pytorch-training<3" \ psutil==5.6.7 \ "pillow>=8.2.2,<8.4" \ && pip uninstall -y torch \ && pip install --no-cache-dir -U ${PT_TRAINING_URL} \ && pip uninstall -y torchvision \ && pip install --no-deps --no-cache-dir -U ${PT_TORCHVISION_URL} # Install extra packages RUN pip install --no-cache-dir -U \ "bokeh>=2.3,<3" \ "imageio>=2.9,<3" \ "opencv-python>=4.3,<5" \ "plotly>=5.1,<6" \ "seaborn>=0.11,<1" \ "shap>=0.39,<1" \ "boto3>1.0,<2.0" \ "sasl<1" \ "thrift_sasl<1" \ "PyHive<1" \ "sparkmagic<1" \ "sagemaker-studio-sparkmagic-lib<1" \ "sagemaker-studio-analytics-extension<1" # Remove python kernel installed by sparkmagic RUN /opt/conda/bin/jupyter-kernelspec remove -fy python3 # Install PT S3 plugin RUN pip install -U ${PT_S3_WHL} RUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/start_with_right_hostname.sh \ && chmod +x /usr/local/bin/deep_learning_container.py RUN HOME_DIR=/root \ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.8/license.txt # Starts framework ENTRYPOINT ["bash", "-m", "start_with_right_hostname.sh"] CMD ["/bin/bash"]