{ "cells": [ { "cell_type": "code", "execution_count": 3, "id": "0162c562-90a2-4dd4-ac05-f43fcc0be8e7", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Writing to /root/.config/pip/pip.conf\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Requirement already satisfied: neuronx-cc==2.* in /opt/conda/lib/python3.8/site-packages (2.6.0.19+3d819e565)\n", "Requirement already satisfied: tensorflow-neuronx in /opt/conda/lib/python3.8/site-packages (2.10.1.2.1.0)\n", "Collecting ipywidgets\n", " Downloading ipywidgets-8.0.6-py3-none-any.whl (138 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.3/138.3 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hCollecting transformers\n", " Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m43.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: islpy<=2022.1.1,>2021.1 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (2022.1.1)\n", "Requirement already satisfied: scipy<=1.7.3 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (1.7.3)\n", "Requirement already satisfied: numpy<=1.21.6,>=1.20 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (1.21.6)\n", "Requirement already satisfied: neuronx-hwm==2.6.0.0 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (2.6.0.0+826e77395)\n", "Requirement already satisfied: requests-unixsocket>=0.1.5 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (0.3.0)\n", "Requirement already satisfied: psutil>=5.6.7 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (5.9.5)\n", "Requirement already satisfied: pgzip>=0.3.0 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (0.3.4)\n", "Requirement already satisfied: networkx<=2.6.3 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (2.6.3)\n", "Requirement already satisfied: python-daemon>=2.2.4 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (3.0.1)\n", "Requirement already satisfied: protobuf<4 in /opt/conda/lib/python3.8/site-packages (from neuronx-cc==2.*) (3.19.6)\n", "Requirement already satisfied: tensorflow-neuron~=2.10.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow-neuronx) (2.10.1.2.8.1.0)\n", "Collecting widgetsnbextension~=4.0.7\n", " Downloading widgetsnbextension-4.0.7-py3-none-any.whl (2.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (5.9.0)\n", "Requirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (6.22.0)\n", "Collecting jupyterlab-widgets~=3.0.7\n", " Downloading jupyterlab_widgets-3.0.7-py3-none-any.whl (198 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.2/198.2 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (8.12.0)\n", "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.8/site-packages (from transformers) (4.65.0)\n", "Collecting huggingface-hub<1.0,>=0.11.0\n", " Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hCollecting filelock\n", " Downloading filelock-3.12.0-py3-none-any.whl (10 kB)\n", "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", " Downloading tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m50.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting regex!=2019.12.17\n", " Downloading regex-2023.5.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (771 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m771.9/771.9 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers) (5.4.1)\n", "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from transformers) (2.30.0)\n", "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from transformers) (23.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n", "Collecting fsspec\n", " Downloading fsspec-2023.4.0-py3-none-any.whl (153 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.0/154.0 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: matplotlib-inline>=0.1 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.6)\n", "Requirement already satisfied: comm>=0.1.1 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.3)\n", "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.3.0)\n", "Requirement already satisfied: nest-asyncio in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.5.6)\n", "Requirement already satisfied: debugpy>=1.6.5 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.6.7)\n", "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (8.2.0)\n", "Requirement already satisfied: tornado>=6.1 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.3.1)\n", "Requirement already satisfied: pyzmq>=20 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (25.0.2)\n", "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.18.2)\n", "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n", "Requirement already satisfied: stack-data in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.2)\n", "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n", "Requirement already satisfied: backcall in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n", "Requirement already satisfied: decorator in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.38)\n", "Requirement already satisfied: pygments>=2.4.0 in /opt/conda/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (2.14.0)\n", "Requirement already satisfied: pytest>=2 in /opt/conda/lib/python3.8/site-packages (from islpy<=2022.1.1,>2021.1->neuronx-cc==2.*) (7.3.1)\n", "Requirement already satisfied: lockfile>=0.10 in /opt/conda/lib/python3.8/site-packages (from python-daemon>=2.2.4->neuronx-cc==2.*) (0.12.2)\n", "Requirement already satisfied: docutils in /opt/conda/lib/python3.8/site-packages (from python-daemon>=2.2.4->neuronx-cc==2.*) (0.19)\n", "Requirement already satisfied: setuptools>=62.4.0 in /opt/conda/lib/python3.8/site-packages (from python-daemon>=2.2.4->neuronx-cc==2.*) (67.7.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->transformers) (3.4)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.8/site-packages (from requests->transformers) (3.1.0)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->transformers) (2.0.2)\n", "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->transformers) (2022.12.7)\n", "Requirement already satisfied: tensorboard-plugin-neuronx in /opt/conda/lib/python3.8/site-packages (from tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.5.26.0)\n", "Requirement already satisfied: tensorflow~=2.10.1.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.10.1)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n", "Requirement already satisfied: importlib-metadata>=4.8.3 in /opt/conda/lib/python3.8/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (6.6.0)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.8/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n", "Requirement already satisfied: platformdirs>=2.5 in /opt/conda/lib/python3.8/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel>=4.5.1->ipywidgets) (3.2.0)\n", "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.6)\n", "Requirement already satisfied: tomli>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from pytest>=2->islpy<=2022.1.1,>2021.1->neuronx-cc==2.*) (2.0.1)\n", "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /opt/conda/lib/python3.8/site-packages (from pytest>=2->islpy<=2022.1.1,>2021.1->neuronx-cc==2.*) (1.1.1)\n", "Requirement already satisfied: iniconfig in /opt/conda/lib/python3.8/site-packages (from pytest>=2->islpy<=2022.1.1,>2021.1->neuronx-cc==2.*) (2.0.0)\n", "Requirement already satisfied: pluggy<2.0,>=0.12 in /opt/conda/lib/python3.8/site-packages (from pytest>=2->islpy<=2022.1.1,>2021.1->neuronx-cc==2.*) (1.0.0)\n", "Requirement already satisfied: tensorflow-estimator<2.11,>=2.10.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.10.0)\n", "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.2.0)\n", "Requirement already satisfied: tensorboard<2.11,>=2.10 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.10.1)\n", "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (3.8.0)\n", "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (3.3.0)\n", "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.16.0)\n", "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.3.0)\n", "Requirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.1.2)\n", "Requirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (16.0.0)\n", "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.15.0)\n", "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.4.0)\n", "Requirement already satisfied: flatbuffers>=2.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (23.3.3)\n", "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.32.0)\n", "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.6.3)\n", "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.54.0)\n", "Requirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.4.0)\n", "Requirement already satisfied: keras<2.11,>=2.10.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.10.0)\n", "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (1.2.0)\n", "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.2.1)\n", "Requirement already satisfied: pure-eval in /opt/conda/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n", "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.8/site-packages (from astunparse>=1.6.0->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.40.0)\n", "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.8/site-packages (from importlib-metadata>=4.8.3->jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (3.15.0)\n", "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (3.4.3)\n", "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.3.3)\n", "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.6.1)\n", "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.4.6)\n", "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.8.1)\n", "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.17.3)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.3.0)\n", "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (5.3.0)\n", "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (4.9)\n", "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (1.3.1)\n", "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (2.1.2)\n", "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (0.5.0)\n", "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.1.0->tensorflow-neuron~=2.10.1->tensorflow-neuronx) (3.2.2)\n", "Installing collected packages: tokenizers, widgetsnbextension, regex, jupyterlab-widgets, fsspec, filelock, huggingface-hub, transformers, ipywidgets\n", "Successfully installed filelock-3.12.0 fsspec-2023.4.0 huggingface-hub-0.14.1 ipywidgets-8.0.6 jupyterlab-widgets-3.0.7 regex-2023.5.5 tokenizers-0.13.3 transformers-4.28.1 widgetsnbextension-4.0.7\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m" ] } ], "source": [ "!python -m pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com\n", "!pip install neuronx-cc==2.* tensorflow-neuronx ipywidgets transformers\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "b26312cb-01fd-45bc-bae8-a3d394809e80", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Downloading (…)okenizer_config.json: 100%|██████████| 29.0/29.0 [00:00<00:00, 8.11kB/s]\n", "Downloading (…)lve/main/config.json: 100%|██████████| 433/433 [00:00<00:00, 152kB/s]\n", "Downloading (…)solve/main/vocab.txt: 100%|██████████| 213k/213k [00:00<00:00, 8.91MB/s]\n", "Downloading (…)/main/tokenizer.json: 100%|██████████| 436k/436k [00:00<00:00, 13.1MB/s]\n", "Downloading pytorch_model.bin: 100%|██████████| 433M/433M [00:01<00:00, 237MB/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] } ], "source": [ "import torch\n", "import torch_neuronx\n", "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n", "import transformers\n", "\n", "\n", "def encode(tokenizer, *inputs, max_length=128, batch_size=1):\n", " tokens = tokenizer.encode_plus(\n", " *inputs,\n", " max_length=max_length,\n", " padding='max_length',\n", " truncation=True,\n", " return_tensors=\"pt\"\n", " )\n", " return (\n", " torch.repeat_interleave(tokens['input_ids'], batch_size, 0),\n", " torch.repeat_interleave(tokens['attention_mask'], batch_size, 0),\n", " torch.repeat_interleave(tokens['token_type_ids'], batch_size, 0),\n", " )\n", "\n", "\n", "# Create the tokenizer and model\n", "name = \"bert-base-cased-finetuned-mrpc\"\n", "tokenizer = AutoTokenizer.from_pretrained(name)\n", "model = AutoModelForSequenceClassification.from_pretrained(name, torchscript=True)\n", "\n", "# Set up some example inputs\n", "sequence_0 = \"The company HuggingFace is based in New York City\"\n", "sequence_1 = \"Apples are especially bad for your health\"\n", "sequence_2 = \"HuggingFace's headquarters are situated in Manhattan\"\n", "\n", "paraphrase = encode(tokenizer, sequence_0, sequence_2)\n", "not_paraphrase = encode(tokenizer, sequence_0, sequence_1)\n", "\n", "# Run the original PyTorch BERT model on CPU\n", "cpu_paraphrase_logits = model(*paraphrase)[0]\n", "cpu_not_paraphrase_logits = model(*not_paraphrase)[0]\n", "\n", "# Compile the model for Neuron\n", "model_neuron = torch_neuronx.trace(model, paraphrase)\n", "\n", "# Save the TorchScript for inference deployment\n", "filename = 'model.pt'\n", "torch.jit.save(model_neuron, filename)" ] }, { "cell_type": "code", "execution_count": 9, "id": "cb9163be-ca2a-4e5c-92d0-1f849a5b12b0", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU paraphrase logits: [[-0.34945598 1.9003887 ]]\n", "Neuron paraphrase logits: [[-0.34909704 1.8992746 ]]\n", "CPU not-paraphrase logits: [[ 0.5386365 -2.2197142]]\n", "Neuron not-paraphrase logits: [[ 0.537705 -2.2180324]]\n" ] } ], "source": [ "# Load the TorchScript compiled model\n", "model_neuron = torch.jit.load(filename)\n", "\n", "# Verify the TorchScript works on both example inputs\n", "neuron_paraphrase_logits = model_neuron(*paraphrase)[0]\n", "neuron_not_paraphrase_logits = model_neuron(*not_paraphrase)[0]\n", "\n", "# Compare the results\n", "print('CPU paraphrase logits: ', cpu_paraphrase_logits.detach().numpy())\n", "print('Neuron paraphrase logits: ', neuron_paraphrase_logits.detach().numpy())\n", "print('CPU not-paraphrase logits: ', cpu_not_paraphrase_logits.detach().numpy())\n", "print('Neuron not-paraphrase logits: ', neuron_not_paraphrase_logits.detach().numpy())" ] }, { "cell_type": "code", "execution_count": 12, "id": "995e0eb5-5b7a-43c2-9310-6cdb14da88ec", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Filename: model.pt\n", "Batch Size: 1\n", "Batches: 20000\n", "Inferences: 20000\n", "Threads: 2\n", "Models: 2\n", "Duration: 9.944\n", "Throughput: 2011.203\n", "Latency P50: 0.994\n", "Latency P95: 1.017\n", "Latency P99: 1.045\n" ] } ], "source": [ "import time\n", "import concurrent.futures\n", "import numpy as np\n", "\n", "\n", "def benchmark(filename, example, n_models=2, n_threads=2, batches_per_thread=10000):\n", " \"\"\"\n", " Record performance statistics for a serialized model and its input example.\n", "\n", " Arguments:\n", " filename: The serialized torchscript model to load for benchmarking.\n", " example: An example model input.\n", " n_models: The number of models to load.\n", " n_threads: The number of simultaneous threads to execute inferences on.\n", " batches_per_thread: The number of example batches to run per thread.\n", "\n", " Returns:\n", " A dictionary of performance statistics.\n", " \"\"\"\n", "\n", " # Load models\n", " models = [torch.jit.load(filename) for _ in range(n_models)]\n", "\n", " # Warmup\n", " for _ in range(8):\n", " for model in models:\n", " model(*example)\n", "\n", " latencies = []\n", "\n", " # Thread task\n", " def task(model):\n", " for _ in range(batches_per_thread):\n", " start = time.time()\n", " model(*example)\n", " finish = time.time()\n", " latencies.append((finish - start) * 1000)\n", "\n", " # Submit tasks\n", " begin = time.time()\n", " with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as pool:\n", " for i in range(n_threads):\n", " pool.submit(task, models[i % len(models)])\n", " end = time.time()\n", "\n", " # Compute metrics\n", " boundaries = [50, 95, 99]\n", " percentiles = {}\n", "\n", " for boundary in boundaries:\n", " name = f'latency_p{boundary}'\n", " percentiles[name] = np.percentile(latencies, boundary)\n", " duration = end - begin\n", " batch_size = 0\n", " for tensor in example:\n", " if batch_size == 0:\n", " batch_size = tensor.shape[0]\n", " inferences = len(latencies) * batch_size\n", " throughput = inferences / duration\n", "\n", " # Metrics\n", " metrics = {\n", " 'filename': str(filename),\n", " 'batch_size': batch_size,\n", " 'batches': len(latencies),\n", " 'inferences': inferences,\n", " 'threads': n_threads,\n", " 'models': n_models,\n", " 'duration': duration,\n", " 'throughput': throughput,\n", " **percentiles,\n", " }\n", "\n", " display(metrics)\n", "\n", "\n", "def display(metrics):\n", " \"\"\"\n", " Display the metrics produced by `benchmark` function.\n", "\n", " Args:\n", " metrics: A dictionary of performance statistics.\n", " \"\"\"\n", " pad = max(map(len, metrics)) + 1\n", " for key, value in metrics.items():\n", "\n", " parts = key.split('_')\n", " parts = list(map(str.title, parts))\n", " title = ' '.join(parts) + \":\"\n", "\n", " if isinstance(value, float):\n", " value = f'{value:0.3f}'\n", "\n", " print(f'{title :<{pad}} {value}')\n", "\n", "\n", "# Benchmark BERT on Neuron\n", "benchmark(filename, paraphrase)" ] }, { "cell_type": "code", "execution_count": null, "id": "24cdb11e-1c28-4f9b-9158-3f56cfdc5584", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.16" } }, "nbformat": 4, "nbformat_minor": 5 }