ARG PYTHON=python3 ARG PYTHON_VERSION=3.9.16 ARG MAMBA_VERSION=22.11.1-2 # PyTorch Binaries ARG PT_EC2_TRAINING_URL=https://aws-pytorch-unified-cicd-binaries.s3.us-west-2.amazonaws.com/r1.13.1_ec2/20221217-194030/54406b8eed7fbd61be629cb06229dfb7b6b2954e/torch-1.13.1%2Bcpu-cp39-cp39-linux_x86_64.whl ARG PT_SM_TRAINING_URL=https://aws-pytorch-unified-cicd-binaries.s3.us-west-2.amazonaws.com/r1.13.1_sm/20230104-200143/4ad54e91be2ccb431cf26e7d7d1a2de10e8d82ac/torch-1.13.1%2Bcpu-cp39-cp39-linux_x86_64.whl ARG PT_TORCHVISION_URL=https://download.pytorch.org/whl/cpu/torchvision-0.14.1%2Bcpu-cp39-cp39-linux_x86_64.whl ARG PT_TORCHAUDIO_URL=https://download.pytorch.org/whl/cpu/torchaudio-0.13.1%2Bcpu-cp39-cp39-linux_x86_64.whl ARG PT_TORCHDATA_URL=https://download.pytorch.org/whl/test/torchdata-0.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl FROM ubuntu:20.04 AS base_image ENV DEBIAN_FRONTEND=noninteractive \ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" RUN apt-get update \ && apt-get upgrade -y \ && apt-get autoremove -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* FROM base_image AS common LABEL maintainer="Amazon AI" LABEL dlc_major_version="1" ARG PYTHON ARG PYTHON_VERSION ARG OPEN_MPI_VERSION=4.0.1 ARG MAMBA_VERSION # This arg required to stop docker build waiting for region configuration while installing tz data from ubuntu 20 ARG DEBIAN_FRONTEND=noninteractive # Python won’t try to write .pyc or .pyo files on the import of source modules # Force stdin, stdout and stderr to be totally unbuffered. Good for logging ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/conda/lib" ENV PATH=/opt/conda/bin:$PATH ENV DGLBACKEND=pytorch ENV DLC_CONTAINER_TYPE=training WORKDIR / RUN apt-get update \ # TODO: Remove systemd upgrade once it is updated in base image && apt-get -y upgrade --only-upgrade systemd \ && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ cmake \ curl \ emacs \ git \ jq \ libcurl4-openssl-dev \ libglib2.0-0 \ libgl1-mesa-glx \ libsm6 \ libssl-dev \ libxext6 \ libxrender-dev \ software-properties-common \ unzip \ vim \ wget \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean # Install Open MPI RUN wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-$OPEN_MPI_VERSION.tar.gz \ && gunzip -c openmpi-$OPEN_MPI_VERSION.tar.gz | tar xf - \ && cd openmpi-$OPEN_MPI_VERSION \ && ./configure --prefix=/home/.openmpi \ && make all install \ && cd .. \ && rm openmpi-$OPEN_MPI_VERSION.tar.gz \ && rm -rf openmpi-$OPEN_MPI_VERSION # The ENV variables declared below are changed in the previous section # Grouping these ENV variables in the first section causes # ompi_info to fail. This is only observed in CPU containers ENV PATH="$PATH:/home/.openmpi/bin" ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/.openmpi/lib/" RUN ompi_info --parsable --all | grep mpi_built_with_cuda_support:value # Install OpenSSH for MPI to communicate between containers, allow OpenSSH to talk to containers without asking for confirmation RUN apt-get update \ && apt-get install -y --no-install-recommends openssh-client openssh-server \ && mkdir -p /var/run/sshd \ && cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean # Configure OpenSSH so that nodes can communicate with each other RUN mkdir -p /var/run/sshd \ && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd RUN rm -rf /root/.ssh/ \ && mkdir -p /root/.ssh/ \ && ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config # for conda ssl verification ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt RUN curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-x86_64.sh \ && chmod +x ~/mambaforge.sh \ && ~/mambaforge.sh -b -p /opt/conda \ && rm ~/mambaforge.sh \ && /opt/conda/bin/conda update -y conda \ && /opt/conda/bin/conda install -y -c conda-forge \ python=$PYTHON_VERSION \ cython \ mkl \ mkl-include \ parso \ typing \ h5py \ requests \ pyopenssl \ # Below 2 are included in miniconda base, but not mamba so need to install conda-content-trust \ charset-normalizer \ # needed by fastai.distributed accelerate \ # pin dgl to 0.9.1 as upstream test has not caught up, currently having accuracy error && /opt/conda/bin/conda install -c dglteam -y dgl=0.9.1 \ && /opt/conda/bin/conda install -c fastai fastai \ # fastai conda has a dependency chain of fastai -> spacy -> pathy -> dataclasses # conda pathy installs dataclasses regardless of py version: https://github.com/conda-forge/pathy-feedstock/blob/1e6da43da5c8e11e09c063c79b09236bbacb7780/recipe/meta.yaml#L29 # pypi solves this issue: https://github.com/justindujardin/pathy/blob/master/requirements.txt#L3 # dataclasses package is not needed for py>=37 # install plainly will oversize the image && pip uninstall -y dataclasses \ && /opt/conda/bin/conda clean -ya # remove the default mamba installation RUN rm -rf /root/micromamba/ # Conda installs links for libtinfo.so.6 and libtinfo.so.6.2 both # Which causes "/opt/conda/lib/libtinfo.so.6: no version information available" warning # Removing link for libtinfo.so.6. This change is needed only for ubuntu 20.04-conda, and can be reverted # once conda fixes the issue RUN rm -rf /opt/conda/lib/libtinfo.so.6 RUN pip install --no-cache-dir --upgrade pip --trusted-host pypi.org --trusted-host files.pythonhosted.org \ && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 \ && pip install --no-cache-dir -U \ "awscli>1.27,<2" \ boto3 \ "click==8.1.2,<9" \ "cryptography>39,<40" \ ipython \ "opencv-python>=4.6.0,<4.7" \ packaging \ Pillow \ "psutil>=5.9.4,<5.10" \ "pyyaml>=5.4,<5.5" \ "scipy>=1.9.3,<2" COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.13/license.txt # Removing the cache as it is needed for security verification RUN rm -rf /root/.cache | true ######################################################## # _____ ____ ____ ___ # | ____/ ___|___ \ |_ _|_ __ ___ __ _ __ _ ___ # | _|| | __) | | || '_ ` _ \ / _` |/ _` |/ _ \ # | |__| |___ / __/ | || | | | | | (_| | (_| | __/ # |_____\____|_____| |___|_| |_| |_|\__,_|\__, |\___| # |___/ # ____ _ # | _ \ ___ ___(_)_ __ ___ # | |_) / _ \/ __| | '_ \ / _ \ # | _ < __/ (__| | |_) | __/ # |_| \_\___|\___|_| .__/ \___| # |_| ######################################################## FROM common AS ec2 ARG PYTHON # PyTorch Binaries ARG PT_EC2_TRAINING_URL ARG PT_TORCHVISION_URL ARG PT_TORCHAUDIO_URL ARG PT_TORCHDATA_URL # Install AWS-PyTorch and other torch packages RUN pip uninstall -y torch torchvision torchaudio torchdata \ && pip install --no-cache-dir -U ${PT_EC2_TRAINING_URL} ${PT_TORCHVISION_URL} ${PT_TORCHAUDIO_URL} ${PT_TORCHDATA_URL} RUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt RUN HOME_DIR=/root \ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* \ && rm -rf /tmp/tmp* # Removing the cache as it is needed for security verification RUN rm -rf /root/.cache | true # Starts framework CMD ["/bin/bash"] ################################################################# # ____ __ __ _ # / ___| __ _ __ _ ___| \/ | __ _| | _____ _ __ # \___ \ / _` |/ _` |/ _ \ |\/| |/ _` | |/ / _ \ '__| # ___) | (_| | (_| | __/ | | | (_| | < __/ | # |____/ \__,_|\__, |\___|_| |_|\__,_|_|\_\___|_| # |___/ # ___ ____ _ # |_ _|_ __ ___ __ _ __ _ ___ | _ \ ___ ___(_)_ __ ___ # | || '_ ` _ \ / _` |/ _` |/ _ \ | |_) / _ \/ __| | '_ \ / _ \ # | || | | | | | (_| | (_| | __/ | _ < __/ (__| | |_) | __/ # |___|_| |_| |_|\__,_|\__, |\___| |_| \_\___|\___|_| .__/ \___| # |___/ |_| ################################################################# FROM common AS sagemaker LABEL maintainer="Amazon AI" LABEL dlc_major_version="1" ARG PYTHON # The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating # the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. ARG SMDEBUG_VERSION=1.0.34 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main ARG PT_SM_TRAINING_URL ARG PT_TORCHVISION_URL ARG PT_TORCHAUDIO_URL ARG PT_TORCHDATA_URL # Install AWS-PyTorch and other torch packages RUN pip uninstall -y torch torchvision torchaudio torchdata \ && pip install --no-cache-dir -U ${PT_SM_TRAINING_URL} ${PT_TORCHVISION_URL} ${PT_TORCHAUDIO_URL} ${PT_TORCHDATA_URL} # needed by torchdata RUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt WORKDIR / # Copy workaround script for incorrect hostname COPY changehostname.c / COPY start_with_right_hostname.sh /usr/local/bin/start_with_right_hostname.sh RUN chmod +x /usr/local/bin/start_with_right_hostname.sh # Install scikit-learn and pandas RUN conda install -y -c conda-forge \ scikit-learn \ pandas RUN pip install --no-cache-dir --upgrade pip --trusted-host pypi.org --trusted-host files.pythonhosted.org \ && pip install --no-cache-dir -U \ # disable smdebug pip install until available stable smdebug releases # smdebug==${SMDEBUG_VERSION} \ smclarify \ "sagemaker>=2,<3" \ sagemaker-experiments==0.* \ sagemaker-pytorch-training # Install smdebug from souce RUN cd /tmp \ && git clone https://github.com/awslabs/sagemaker-debugger --branch ${SMDEBUG_VERSION} --depth 1 --single-branch \ && cd sagemaker-debugger \ && pip install . \ && rm -rf /tmp/* # Install extra packages RUN pip install --no-cache-dir -U \ "bokeh>=3.0.1,<4" \ "imageio>=2.22,<3" \ "opencv-python>=4.6,<5" \ "plotly>=5.11,<6" \ "seaborn>=0.12,<1" \ "numba>=0.56.4,<0.57" \ "shap>=0.41,<1" RUN HOME_DIR=/root \ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* \ && rm -rf /tmp/tmp* # Removing the cache as it is needed for security verification RUN rm -rf /root/.cache | true ENTRYPOINT ["bash", "-m", "start_with_right_hostname.sh"] CMD ["/bin/bash"]