{ "cells": [ { "cell_type": "markdown", "id": "8f3dd71b-f63a-4106-b323-67be004b482e", "metadata": {}, "source": [ "# Setup for Nvidai NeMo with SageMaker \n", "* Container: codna_python3" ] }, { "cell_type": "markdown", "id": "aaeaff49-a119-4a0c-b48a-f66ab721c458", "metadata": {}, "source": [ "## AutoReload" ] }, { "cell_type": "code", "execution_count": 1, "id": "9f8f8f99-aaa0-42f9-8695-c8bd1884504c", "metadata": { "tags": [] }, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "markdown", "id": "78872034-4b66-45e4-983f-726a94459873", "metadata": { "tags": [] }, "source": [ "## 0. Install packages" ] }, { "cell_type": "code", "execution_count": 2, "id": "ee290bd6-1b9b-44d5-8746-14412ea3ca96", "metadata": { "tags": [] }, "outputs": [], "source": [ "install_needed = True # should only be True once\n", "# install_needed = False" ] }, { "cell_type": "code", "execution_count": 3, "id": "6594574f-22e6-4237-a573-5598e0246727", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Already revised\n" ] } ], "source": [ "%%bash\n", "#!/bin/bash\n", "\n", "DAEMON_PATH=\"/etc/docker\"\n", "MEMORY_SIZE=10G\n", "\n", "FLAG=$(cat $DAEMON_PATH/daemon.json | jq 'has(\"data-root\")')\n", "# echo $FLAG\n", "\n", "if [ \"$FLAG\" == true ]; then\n", " echo \"Already revised\"\n", "else\n", " echo \"Add data-root and default-shm-size=$MEMORY_SIZE\"\n", " sudo cp $DAEMON_PATH/daemon.json $DAEMON_PATH/daemon.json.bak\n", " sudo cat $DAEMON_PATH/daemon.json.bak | jq '. += {\"data-root\":\"/home/ec2-user/SageMaker/.container/docker\",\"default-shm-size\":\"'$MEMORY_SIZE'\"}' | sudo tee $DAEMON_PATH/daemon.json > /dev/null\n", " sudo service docker restart\n", " echo \"Docker Restart\"\n", "fi" ] }, { "cell_type": "code", "execution_count": 4, "id": "f21840e5-9ecf-4e9c-9736-d2174cd413e2", "metadata": { "collapsed": true, "jupyter": { "outputs_hidden": true }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "installing deps and restarting kernel\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (22.3.1)\n", "Collecting pip\n", " Downloading pip-23.0.1-py3-none-any.whl (2.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: pip\n", " Attempting uninstall: pip\n", " Found existing installation: pip 22.3.1\n", " Uninstalling pip-22.3.1:\n", " Successfully uninstalled pip-22.3.1\n", "Successfully installed pip-23.0.1\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Collecting smdebug\n", " Downloading smdebug-1.0.12-py2.py3-none-any.whl (270 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m270.1/270.1 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting sagemaker-experiments\n", " Downloading sagemaker_experiments-0.1.43-py3-none-any.whl (42 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.6/42.6 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from smdebug) (21.3)\n", "Requirement already satisfied: numpy>=1.16.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from smdebug) (1.23.5)\n", "Requirement already satisfied: protobuf>=3.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from smdebug) (3.20.2)\n", "Requirement already satisfied: boto3>=1.10.32 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from smdebug) (1.26.71)\n", "Collecting pyinstrument==3.4.2\n", " Downloading pyinstrument-3.4.2-py2.py3-none-any.whl (83 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.3/83.3 kB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyinstrument-cext>=0.2.2\n", " Downloading pyinstrument_cext-0.2.4-cp39-cp39-manylinux2010_x86_64.whl (20 kB)\n", "Requirement already satisfied: botocore<1.30.0,>=1.29.71 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3>=1.10.32->smdebug) (1.29.71)\n", "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3>=1.10.32->smdebug) (1.0.1)\n", "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3>=1.10.32->smdebug) (0.6.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging->smdebug) (3.0.9)\n", "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.71->boto3>=1.10.32->smdebug) (1.26.8)\n", "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.71->boto3>=1.10.32->smdebug) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.30.0,>=1.29.71->boto3>=1.10.32->smdebug) (1.16.0)\n", "Installing collected packages: pyinstrument-cext, pyinstrument, smdebug, sagemaker-experiments\n", "Successfully installed pyinstrument-3.4.2 pyinstrument-cext-0.2.4 sagemaker-experiments-0.1.43 smdebug-1.0.12\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Requirement already satisfied: sagemaker in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (2.132.0)\n", "Collecting sagemaker\n", " Downloading sagemaker-2.140.1.tar.gz (684 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m684.5/684.5 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: attrs<23,>=20.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (22.2.0)\n", "Requirement already satisfied: boto3<2.0,>=1.26.28 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.26.71)\n", "Requirement already satisfied: google-pasta in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.2.0)\n", "Requirement already satisfied: numpy<2.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.23.5)\n", "Requirement already satisfied: protobuf<4.0,>=3.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (3.20.2)\n", "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.1.5)\n", "Requirement already satisfied: smdebug_rulesconfig==1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.0.1)\n", "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (4.13.0)\n", "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (21.3)\n", "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.4.4)\n", "Requirement already satisfied: pathos in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.3.0)\n", "Requirement already satisfied: schema in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.7.5)\n", "Requirement already satisfied: botocore<1.30.0,>=1.29.71 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.28->sagemaker) (1.29.71)\n", "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.28->sagemaker) (0.6.0)\n", "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.28->sagemaker) (1.0.1)\n", "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.11.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging>=20.0->sagemaker) (3.0.9)\n", "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.16.0)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->sagemaker) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->sagemaker) (2022.7)\n", "Requirement already satisfied: multiprocess>=0.70.14 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.70.14)\n", "Requirement already satisfied: pox>=0.3.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.2)\n", "Requirement already satisfied: ppft>=1.7.6.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (1.7.6.6)\n", "Requirement already satisfied: dill>=0.3.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.6)\n", "Requirement already satisfied: contextlib2>=0.5.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from schema->sagemaker) (21.6.0)\n", "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.71->boto3<2.0,>=1.26.28->sagemaker) (1.26.8)\n", "Building wheels for collected packages: sagemaker\n", " Building wheel for sagemaker (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for sagemaker: filename=sagemaker-2.140.1-py2.py3-none-any.whl size=925381 sha256=31dac86efa555305ae1650de0ec1fb19a85debcdcc4891404cf2bd40e1d87820\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/73/18/c5/3ad3801205b996b9ef1dafcdb5fd09d701fb9290c52a066b74\n", "Successfully built sagemaker\n", "Installing collected packages: sagemaker\n", " Attempting uninstall: sagemaker\n", " Found existing installation: sagemaker 2.132.0\n", " Uninstalling sagemaker-2.132.0:\n", " Successfully uninstalled sagemaker-2.132.0\n", "Successfully installed sagemaker-2.140.1\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Collecting datasets\n", " Downloading datasets-2.10.1-py3-none-any.whl (469 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting transformers\n", " Downloading transformers-4.27.2-py3-none-any.whl (6.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (2022.11.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (4.63.2)\n", "Collecting xxhash\n", " Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (2.28.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (5.4.1)\n", "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (0.3.6)\n", "Requirement already satisfied: pyarrow>=6.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (10.0.1)\n", "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (21.3)\n", "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (1.4.4)\n", "Requirement already satisfied: multiprocess in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (0.70.14)\n", "Requirement already satisfied: aiohttp in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (3.8.3)\n", "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from datasets) (1.23.5)\n", "Collecting responses<0.19\n", " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", "Collecting huggingface-hub<1.0.0,>=0.2.0\n", " Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", " Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m45.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from transformers) (2022.10.31)\n", "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from transformers) (3.6.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (1.8.2)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.3)\n", "Requirement already satisfied: attrs>=17.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (22.2.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->datasets) (6.0.4)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging->datasets) (3.0.9)\n", "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (3.4)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (1.26.8)\n", "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (2022.12.7)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->datasets) (2022.7)\n", "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n", "Installing collected packages: tokenizers, xxhash, responses, huggingface-hub, transformers, datasets\n", "Successfully installed datasets-2.10.1 huggingface-hub-0.13.3 responses-0.18.0 tokenizers-0.13.2 transformers-4.27.2 xxhash-3.2.0\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Collecting wget\n", " Downloading wget-3.2.zip (10 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting omegaconf\n", " Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: text-unidecode in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (1.3)\n", "Collecting sox\n", " Downloading sox-1.4.1-py2.py3-none-any.whl (39 kB)\n", "Requirement already satisfied: PyYAML>=5.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from omegaconf) (5.4.1)\n", "Collecting antlr4-python3-runtime==4.9.*\n", " Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m24.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: numpy>=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sox) (1.23.5)\n", "Building wheels for collected packages: wget, antlr4-python3-runtime\n", " Building wheel for wget (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9656 sha256=92c5c14faca2d013fde6c77959f97d208f3d0deb08e0ac11b3404193989e528a\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/04/5f/3e/46cc37c5d698415694d83f607f833f83f0149e49b3af9d0f38\n", " Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=21092ddbd39689899148887062bdd0ede51799fee89895e6d3cc29d980ccfa6b\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/23/cf/80/f3efa822e6ab23277902ee9165fe772eeb1dfb8014f359020a\n", "Successfully built wget antlr4-python3-runtime\n", "Installing collected packages: wget, antlr4-python3-runtime, sox, omegaconf\n", "Successfully installed antlr4-python3-runtime-4.9.3 omegaconf-2.3.0 sox-1.4.1 wget-3.2\n", "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n", "amzn2-core | 3.7 kB 00:00 \n", "amzn2extra-docker | 3.0 kB 00:00 \n", "amzn2extra-kernel-5.10 | 3.0 kB 00:00 \n", "amzn2extra-python3.8 | 3.0 kB 00:00 \n", "centos-extras | 2.9 kB 00:00 \n", "copr:copr.fedorainfracloud.org:vbatts:shadow-utils-newxi | 3.3 kB 00:00 \n", "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n", "Trying other mirror.\n", "libnvidia-container/x86_64/signature | 833 B 00:00 \n", "libnvidia-container/x86_64/signature | 2.1 kB 00:00 !!! \n", "neuron | 2.9 kB 00:00 \n", "(1/2): neuron/primary_db | 111 kB 00:00 \n", "(2/2): libnvidia-container/x86_64/primary | 26 kB 00:00 \n", "libnvidia-container 167/167\n", "61 packages excluded due to repository priority protections\n", "Resolving Dependencies\n", "--> Running transaction check\n", "---> Package sox.x86_64 0:14.4.1-7.amzn2 will be installed\n", "--> Processing Dependency: libpulse.so.0(PULSE_0)(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libpulse-simple.so.0(PULSE_0)(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libao.so.4(LIBAO4_1.1.0)(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libwavpack.so.1()(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libpulse.so.0()(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libpulse-simple.so.0()(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Processing Dependency: libao.so.4()(64bit) for package: sox-14.4.1-7.amzn2.x86_64\n", "--> Running transaction check\n", "---> Package libao.x86_64 0:1.1.0-8.amzn2.0.2 will be installed\n", "---> Package pulseaudio-libs.x86_64 0:10.0-3.amzn2.0.3 will be installed\n", "--> Processing Dependency: libasyncns.so.0()(64bit) for package: pulseaudio-libs-10.0-3.amzn2.0.3.x86_64\n", "---> Package wavpack.x86_64 0:4.60.1-9.amzn2.0.1 will be installed\n", "--> Running transaction check\n", "---> Package libasyncns.x86_64 0:0.8-7.amzn2.0.2 will be installed\n", "--> Finished Dependency Resolution\n", "\n", "Dependencies Resolved\n", "\n", "================================================================================\n", " Package Arch Version Repository Size\n", "================================================================================\n", "Installing:\n", " sox x86_64 14.4.1-7.amzn2 amzn2-core 401 k\n", "Installing for dependencies:\n", " libao x86_64 1.1.0-8.amzn2.0.2 amzn2-core 72 k\n", " libasyncns x86_64 0.8-7.amzn2.0.2 amzn2-core 26 k\n", " pulseaudio-libs x86_64 10.0-3.amzn2.0.3 amzn2-core 648 k\n", " wavpack x86_64 4.60.1-9.amzn2.0.1 amzn2-core 132 k\n", "\n", "Transaction Summary\n", "================================================================================\n", "Install 1 Package (+4 Dependent packages)\n", "\n", "Total download size: 1.2 M\n", "Installed size: 4.8 M\n", "Downloading packages:\n", "(1/5): libasyncns-0.8-7.amzn2.0.2.x86_64.rpm | 26 kB 00:00 \n", "(2/5): pulseaudio-libs-10.0-3.amzn2.0.3.x86_64.rpm | 648 kB 00:00 \n", "(3/5): libao-1.1.0-8.amzn2.0.2.x86_64.rpm | 72 kB 00:00 \n", "(4/5): wavpack-4.60.1-9.amzn2.0.1.x86_64.rpm | 132 kB 00:00 \n", "(5/5): sox-14.4.1-7.amzn2.x86_64.rpm | 401 kB 00:00 \n", "--------------------------------------------------------------------------------\n", "Total 3.2 MB/s | 1.2 MB 00:00 \n", "Running transaction check\n", "Running transaction test\n", "Transaction test succeeded\n", "Running transaction\n", "Warning: RPMDB altered outside of yum.\n", " Installing : wavpack-4.60.1-9.amzn2.0.1.x86_64 1/5 \n", " Installing : libasyncns-0.8-7.amzn2.0.2.x86_64 2/5 \n", " Installing : pulseaudio-libs-10.0-3.amzn2.0.3.x86_64 3/5 \n", " Installing : libao-1.1.0-8.amzn2.0.2.x86_64 4/5 \n", " Installing : sox-14.4.1-7.amzn2.x86_64 5/5 \n", " Verifying : libasyncns-0.8-7.amzn2.0.2.x86_64 1/5 \n", " Verifying : sox-14.4.1-7.amzn2.x86_64 2/5 \n", " Verifying : libao-1.1.0-8.amzn2.0.2.x86_64 3/5 \n", " Verifying : pulseaudio-libs-10.0-3.amzn2.0.3.x86_64 4/5 \n", " Verifying : wavpack-4.60.1-9.amzn2.0.1.x86_64 5/5 \n", "\n", "Installed:\n", " sox.x86_64 0:14.4.1-7.amzn2 \n", "\n", "Dependency Installed:\n", " libao.x86_64 0:1.1.0-8.amzn2.0.2 libasyncns.x86_64 0:0.8-7.amzn2.0.2\n", " pulseaudio-libs.x86_64 0:10.0-3.amzn2.0.3 wavpack.x86_64 0:4.60.1-9.amzn2.0.1\n", "\n", "Complete!\n", "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n", "Existing lock /var/run/yum.pid: another copy is running as pid 12444.\n", "Another app is currently holding the yum lock; waiting for it to exit...\n", " The other application is: yum\n", " Memory : 36 M RSS (252 MB VSZ)\n", " Started: Wed Mar 22 09:47:54 2023 - 00:01 ago\n", " State : Running, pid: 12444\n", "amzn2-core | 3.7 kB 00:00 \n", "amzn2extra-docker | 3.0 kB 00:00 \n", "amzn2extra-kernel-5.10 | 3.0 kB 00:00 \n", "amzn2extra-python3.8 | 3.0 kB 00:00 \n", "centos-extras | 2.9 kB 00:00 \n", "copr:copr.fedorainfracloud.org:vbatts:shadow-utils-newxi | 3.3 kB 00:00 \n", "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n", "Trying other mirror.\n", "libnvidia-container/x86_64/signature | 833 B 00:00 \n", "libnvidia-container/x86_64/signature | 2.1 kB 00:00 !!! \n", "neuron | 2.9 kB 00:00 \n", "61 packages excluded due to repository priority protections\n", "Resolving Dependencies\n", "--> Running transaction check\n", "---> Package libsndfile.x86_64 0:1.0.25-12.amzn2.1 will be updated\n", "---> Package libsndfile.x86_64 0:1.0.25-12.amzn2.2 will be an update\n", "--> Finished Dependency Resolution\n", "\n", "Dependencies Resolved\n", "\n", "================================================================================\n", " Package Arch Version Repository Size\n", "================================================================================\n", "Updating:\n", " libsndfile x86_64 1.0.25-12.amzn2.2 amzn2-core 149 k\n", "\n", "Transaction Summary\n", "================================================================================\n", "Upgrade 1 Package\n", "\n", "Total download size: 149 k\n", "Downloading packages:\n", "Delta RPMs disabled because /usr/bin/applydeltarpm not installed.\n", "libsndfile-1.0.25-12.amzn2.2.x86_64.rpm | 149 kB 00:00 \n", "Running transaction check\n", "Running transaction test\n", "Transaction test succeeded\n", "Running transaction\n", " Updating : libsndfile-1.0.25-12.amzn2.2.x86_64 1/2 \n", " Cleanup : libsndfile-1.0.25-12.amzn2.1.x86_64 2/2 \n", " Verifying : libsndfile-1.0.25-12.amzn2.2.x86_64 1/2 \n", " Verifying : libsndfile-1.0.25-12.amzn2.1.x86_64 2/2 \n", "\n", "Updated:\n", " libsndfile.x86_64 0:1.0.25-12.amzn2.2 \n", "\n", "Complete!\n", "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Collecting llvmlite\n", " Downloading llvmlite-0.39.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.6/34.6 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: llvmlite\n", " Attempting uninstall: llvmlite\n", " Found existing installation: llvmlite 0.39.1\n", " Uninstalling llvmlite-0.39.1:\n", " Successfully uninstalled llvmlite-0.39.1\n", "Successfully installed llvmlite-0.39.1\n", "\u001b[33mDEPRECATION: git+https://github.com/NVIDIA/NeMo.git@main#egg=nemo_toolkit[all] contains an egg fragment with a non-PEP 508 name pip 25.0 will enforce this behaviour change. A possible replacement is to use the req @ url syntax, and remove the egg fragment. Discussion can be found at https://github.com/pypa/pip/issues/11617\u001b[0m\u001b[33m\n", "\u001b[0mLooking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n", "Collecting nemo_toolkit[all]\n", " Cloning https://github.com/NVIDIA/NeMo.git (to revision main) to /tmp/pip-install-3glzjuli/nemo-toolkit_d6fe1ee95fe44e2bb315c9aeeea214be\n", " Running command git clone --filter=blob:none --quiet https://github.com/NVIDIA/NeMo.git /tmp/pip-install-3glzjuli/nemo-toolkit_d6fe1ee95fe44e2bb315c9aeeea214be\n", " Resolved https://github.com/NVIDIA/NeMo.git to commit c5466015eec8772b11861bec62024f10596688dd\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: wget in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.2)\n", "Requirement already satisfied: text-unidecode in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.3)\n", "Requirement already satisfied: torch in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.13.1)\n", "Requirement already satisfied: wrapt in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.14.1)\n", "Requirement already satisfied: onnx>=1.7.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.11.0)\n", "Requirement already satisfied: numpy>=1.22 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.23.5)\n", "Collecting setuptools==65.5.1\n", " Downloading setuptools-65.5.1-py3-none-any.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m25.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hCollecting tensorboard\n", " Downloading tensorboard-2.12.0-py3-none-any.whl (5.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m50.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: tqdm>=4.41.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.63.2)\n", "Requirement already satisfied: python-dateutil in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.8.2)\n", "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.0)\n", "Requirement already satisfied: numba in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.56.4)\n", "Requirement already satisfied: huggingface-hub in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.13.3)\n", "Requirement already satisfied: ruamel.yaml in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.17.21)\n", "Collecting pyannote.core\n", " Downloading pyannote.core-5.0.0-py3-none-any.whl (58 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.5/58.5 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: tabulate>=0.8.7 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.0)\n", "Requirement already satisfied: pyyaml<6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.4.1)\n", "Collecting ftfy\n", " Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: ipywidgets in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (8.0.4)\n", "Requirement already satisfied: boto3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.26.71)\n", "Collecting gradio\n", " Downloading gradio-3.23.0-py3-none-any.whl (15.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.8/15.8 MB\u001b[0m \u001b[31m39.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting omegaconf<2.3,>=2.2\n", " Downloading omegaconf-2.2.3-py3-none-any.whl (79 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.3/79.3 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting kaldiio\n", " Downloading kaldiio-2.17.2.tar.gz (24 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting hydra-core<1.3,>=1.2.0\n", " Downloading hydra_core-1.2.0-py3-none-any.whl (151 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m151.1/151.1 kB\u001b[0m \u001b[31m31.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting ijson\n", " Downloading ijson-3.2.0.post0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (112 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.5/112.5 kB\u001b[0m \u001b[31m27.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: sox in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.4.1)\n", "Collecting parameterized\n", " Downloading parameterized-0.8.1-py2.py3-none-any.whl (26 kB)\n", "Collecting black==19.10b0\n", " Downloading black-19.10b0-py36-none-any.whl (97 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.5/97.5 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: scipy>=0.14 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.8.1)\n", "Collecting rapidfuzz\n", " Downloading rapidfuzz-2.13.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hCollecting editdistance\n", " Downloading editdistance-0.6.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m282.4/282.4 kB\u001b[0m \u001b[31m46.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting sacrebleu[ja]\n", " Downloading sacrebleu-2.3.1-py3-none-any.whl (118 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting youtokentome>=1.0.5\n", " Downloading youtokentome-1.0.6.tar.gz (86 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.7/86.7 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting marshmallow\n", " Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting fasttext\n", " Downloading fasttext-0.9.2.tar.gz (68 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m68.8/68.8 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting webdataset<=0.1.62,>=0.1.48\n", " Downloading webdataset-0.1.62-py3-none-any.whl (32 kB)\n", "Requirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.8.1)\n", "Collecting braceexpand\n", " Downloading braceexpand-0.1.7-py2.py3-none-any.whl (5.9 kB)\n", "Collecting jiwer>=2.0.0\n", " Downloading jiwer-3.0.0-py3-none-any.whl (21 kB)\n", "Collecting pypinyin\n", " Downloading pypinyin-0.48.0-py2.py3-none-any.whl (1.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m74.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: flask-restful in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.3.9)\n", "Collecting nemo-text-processing\n", " Downloading nemo_text_processing-0.1.7rc0-py3-none-any.whl (2.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m94.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: textdistance>=4.1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5.0)\n", "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (21.3)\n", "Collecting wandb\n", " Downloading wandb-0.14.0-py3-none-any.whl (2.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: transformers>=4.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.27.2)\n", "Collecting kornia\n", " Downloading kornia-0.6.10-py2.py3-none-any.whl (612 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m612.0/612.0 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting sentencepiece<1.0.0\n", " Downloading sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hCollecting click==8.0.2\n", " Downloading click-8.0.2-py3-none-any.whl (97 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.6/97.6 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyannote.metrics\n", " Downloading pyannote.metrics-3.2.1-py3-none-any.whl (51 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.4/51.4 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pytorch-lightning<=1.9.4,>=1.9.0\n", " Downloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting pydub\n", " Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n", "Collecting inflect\n", " Downloading inflect-6.0.2-py3-none-any.whl (34 kB)\n", "Collecting librosa>=0.9.0\n", " Downloading librosa-0.10.0.post2-py3-none-any.whl (253 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m253.0/253.0 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.4.4)\n", "Requirement already satisfied: isort<6.0.0,>5.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.11.4)\n", "Collecting sphinxcontrib-bibtex\n", " Downloading sphinxcontrib_bibtex-2.5.0-py3-none-any.whl (39 kB)\n", "Collecting sacremoses>=0.0.43\n", " Downloading sacremoses-0.0.53.tar.gz (880 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m880.6/880.6 kB\u001b[0m \u001b[31m82.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting jieba\n", " Downloading jieba-0.42.1.tar.gz (19.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m30.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting sentence-transformers\n", " Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: matplotlib>=3.3.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.5.3)\n", "Collecting attrdict\n", " Downloading attrdict-2.0.1-py2.py3-none-any.whl (9.9 kB)\n", "Collecting pytest-runner\n", " Using cached pytest_runner-6.0.0-py3-none-any.whl (7.2 kB)\n", "Collecting progress>=1.5\n", " Downloading progress-1.6.tar.gz (7.8 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting texterrors\n", " Downloading texterrors-0.4.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m64.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting opencc\n", " Downloading OpenCC-1.1.6-cp39-cp39-manylinux1_x86_64.whl (778 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m778.3/778.3 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pytest in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (7.2.0)\n", "Collecting faiss-cpu\n", " Downloading faiss_cpu-1.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: sphinx in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.1.1)\n", "Collecting gdown\n", " Downloading gdown-4.6.4-py3-none-any.whl (14 kB)\n", "Collecting pypinyin-dict\n", " Downloading pypinyin_dict-0.5.0-py2.py3-none-any.whl (9.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m47.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: h5py in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.7.0)\n", "Collecting pangu\n", " Downloading pangu-4.0.6.1-py3-none-any.whl (6.4 kB)\n", "Collecting kaldi-python-io\n", " Downloading kaldi-python-io-1.2.2.tar.gz (8.8 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting g2p-en\n", " Downloading g2p_en-2.1.0-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hCollecting soundfile\n", " Downloading soundfile-0.12.1-py2.py3-none-any.whl (24 kB)\n", "Collecting torchmetrics>=0.11.0\n", " Downloading torchmetrics-0.11.4-py3-none-any.whl (519 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.2/519.2 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: appdirs in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.4.4)\n", "Requirement already satisfied: pathspec<1,>=0.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.3)\n", "Requirement already satisfied: typed-ast>=1.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.5.4)\n", "Requirement already satisfied: toml>=0.9.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.2)\n", "Requirement already satisfied: attrs>=18.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (22.2.0)\n", "Requirement already satisfied: regex in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (2022.10.31)\n", "Requirement already satisfied: antlr4-python3-runtime==4.9.* in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from hydra-core<1.3,>=1.2.0->nemo_toolkit[all]) (4.9.3)\n", "Collecting jiwer>=2.0.0\n", " Downloading jiwer-2.6.0-py3-none-any.whl (20 kB)\n", " Downloading jiwer-2.5.2-py3-none-any.whl (15 kB)\n", "Requirement already satisfied: typing-extensions>=4.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (4.4.0)\n", "Requirement already satisfied: msgpack>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.0.4)\n", "Requirement already satisfied: pooch<1.7,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.6.0)\n", "Requirement already satisfied: joblib>=0.14 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.2.0)\n", "Collecting soxr>=0.3.2\n", " Downloading soxr-0.3.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m26.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hCollecting audioread>=2.1.9\n", " Downloading audioread-3.0.0.tar.gz (377 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m377.0/377.0 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: decorator>=4.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (5.1.1)\n", "Collecting lazy-loader>=0.1\n", " Downloading lazy_loader-0.2-py3-none-any.whl (8.6 kB)\n", "Requirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (0.11.0)\n", "Requirement already satisfied: pyparsing>=2.2.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (3.0.9)\n", "Requirement already satisfied: fonttools>=4.22.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (4.38.0)\n", "Requirement already satisfied: pillow>=6.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (9.2.0)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (1.4.4)\n", "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from numba->nemo_toolkit[all]) (0.39.1)\n", "Requirement already satisfied: protobuf>=3.12.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from onnx>=1.7.0->nemo_toolkit[all]) (3.20.2)\n", "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from python-dateutil->nemo_toolkit[all]) (1.16.0)\n", "Requirement already satisfied: fsspec[http]>2021.06.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pytorch-lightning<=1.9.4,>=1.9.0->nemo_toolkit[all]) (2022.11.0)\n", "Collecting lightning-utilities>=0.6.0.post0\n", " Downloading lightning_utilities-0.8.0-py3-none-any.whl (20 kB)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from scikit-learn->nemo_toolkit[all]) (3.1.0)\n", "Requirement already satisfied: cffi>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from soundfile->nemo_toolkit[all]) (1.15.1)\n", "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from transformers>=4.0.1->nemo_toolkit[all]) (3.6.0)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from transformers>=4.0.1->nemo_toolkit[all]) (0.13.2)\n", "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from transformers>=4.0.1->nemo_toolkit[all]) (2.28.1)\n", "Requirement already satisfied: botocore<1.30.0,>=1.29.71 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.29.71)\n", "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (0.6.0)\n", "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.0.1)\n", "Requirement already satisfied: pybind11>=2.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from fasttext->nemo_toolkit[all]) (2.9.2)\n", "Requirement already satisfied: pytz in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from flask-restful->nemo_toolkit[all]) (2022.7)\n", "Requirement already satisfied: Flask>=0.8 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from flask-restful->nemo_toolkit[all]) (2.2.2)\n", "Requirement already satisfied: aniso8601>=0.82 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from flask-restful->nemo_toolkit[all]) (9.0.1)\n", "Requirement already satisfied: wcwidth>=0.2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ftfy->nemo_toolkit[all]) (0.2.5)\n", "Collecting distance>=0.1.3\n", " Downloading Distance-0.1.3.tar.gz (180 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m180.3/180.3 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: pydantic>=1.9.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from inflect->nemo_toolkit[all]) (1.10.4)\n", "Requirement already satisfied: beautifulsoup4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from gdown->nemo_toolkit[all]) (4.11.1)\n", "Requirement already satisfied: aiohttp in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from gradio->nemo_toolkit[all]) (3.8.3)\n", "Collecting altair>=4.2.0\n", " Downloading altair-4.2.2-py3-none-any.whl (813 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m813.6/813.6 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting mdit-py-plugins<=0.3.3\n", " Downloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting fastapi\n", " Downloading fastapi-0.95.0-py3-none-any.whl (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.1/57.1 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting uvicorn\n", " Downloading uvicorn-0.21.1-py3-none-any.whl (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.8/57.8 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting websockets>=10.0\n", " Downloading websockets-10.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.5/106.5 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hCollecting ffmpy\n", " Downloading ffmpy-0.3.0.tar.gz (4.8 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting markdown-it-py[linkify]>=2.0.0\n", " Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting python-multipart\n", " Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting httpx\n", " Downloading httpx-0.23.3-py3-none-any.whl (71 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from gradio->nemo_toolkit[all]) (3.1.2)\n", "Collecting semantic-version\n", " Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n", "Collecting orjson\n", " Downloading orjson-3.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (275 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m275.3/275.3 kB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: markupsafe in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from gradio->nemo_toolkit[all]) (2.1.1)\n", "Collecting aiofiles\n", " Downloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\n", "Requirement already satisfied: ipython>=6.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (7.32.0)\n", "Requirement already satisfied: ipykernel>=4.5.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (6.20.1)\n", "Requirement already satisfied: widgetsnbextension~=4.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (4.0.5)\n", "Requirement already satisfied: traitlets>=4.3.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (5.8.1)\n", "Requirement already satisfied: jupyterlab-widgets~=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (3.0.5)\n", "Collecting cdifflib\n", " Downloading cdifflib-1.2.6.tar.gz (11 kB)\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Installing backend dependencies ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25hCollecting pynini==2.1.5\n", " Downloading pynini-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (161.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m161.3/161.3 MB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: Cython>=0.29 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pynini==2.1.5->nemo-text-processing->nemo_toolkit[all]) (0.29.33)\n", "Requirement already satisfied: sortedcontainers>=2.0.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pyannote.core->nemo_toolkit[all]) (2.4.0)\n", "Collecting pyannote.database>=4.0.1\n", " Downloading pyannote.database-4.1.3-py3-none-any.whl (41 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: docopt>=0.6.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (0.6.2)\n", "Requirement already satisfied: sympy>=1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (1.11.1)\n", "Requirement already satisfied: iniconfig in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (2.0.0)\n", "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.0.0)\n", "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.1.0)\n", "Requirement already satisfied: tomli>=1.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (2.0.1)\n", "Requirement already satisfied: ruamel.yaml.clib>=0.2.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ruamel.yaml->nemo_toolkit[all]) (0.2.7)\n", "Collecting lxml\n", " Downloading lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (7.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: colorama in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (0.4.3)\n", "Collecting portalocker\n", " Downloading portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n", "Collecting mecab-python3==1.0.5\n", " Downloading mecab_python3-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (581 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m581.0/581.0 kB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting ipadic<2.0,>=1.0\n", " Downloading ipadic-1.0.0.tar.gz (13.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.4/13.4 MB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: torchvision in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sentence-transformers->nemo_toolkit[all]) (0.14.1)\n", "Requirement already satisfied: docutils<0.20,>=0.14 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.15.2)\n", "Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.1.5)\n", "Requirement already satisfied: importlib-metadata>=4.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (4.13.0)\n", "Requirement already satisfied: sphinxcontrib-qthelp in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.3)\n", "Requirement already satisfied: sphinxcontrib-devhelp in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", "Requirement already satisfied: alabaster<0.8,>=0.7 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.7.12)\n", "Requirement already satisfied: sphinxcontrib-htmlhelp>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.0.0)\n", "Requirement already satisfied: sphinxcontrib-applehelp in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", "Requirement already satisfied: sphinxcontrib-jsmath in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.1)\n", "Requirement already satisfied: snowballstemmer>=1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.2.0)\n", "Requirement already satisfied: babel>=1.3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.11.0)\n", "Requirement already satisfied: Pygments>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.14.0)\n", "Requirement already satisfied: imagesize in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.4.1)\n", "Collecting pybtex-docutils>=1.0.0\n", " Downloading pybtex_docutils-1.0.2-py3-none-any.whl (6.3 kB)\n", "Collecting pybtex>=0.24\n", " Downloading pybtex-0.24.0-py2.py3-none-any.whl (561 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m561.4/561.4 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting google-auth<3,>=1.6.3\n", " Downloading google_auth-2.16.2-py2.py3-none-any.whl (177 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.2/177.2 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting tensorboard-data-server<0.8.0,>=0.7.0\n", " Downloading tensorboard_data_server-0.7.0-py3-none-manylinux2014_x86_64.whl (6.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m47.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting markdown>=2.6.8\n", " Downloading Markdown-3.4.1-py3-none-any.whl (93 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.3/93.3 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting grpcio>=1.48.2\n", " Downloading grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m36.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n", "\u001b[?25hRequirement already satisfied: werkzeug>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from tensorboard->nemo_toolkit[all]) (2.2.2)\n", "Collecting tensorboard-plugin-wit>=1.6.0\n", " Downloading tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m781.3/781.3 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting google-auth-oauthlib<0.5,>=0.4.1\n", " Downloading google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\n", "Requirement already satisfied: wheel>=0.26 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from tensorboard->nemo_toolkit[all]) (0.38.4)\n", "Collecting absl-py>=0.4\n", " Downloading absl_py-1.4.0-py3-none-any.whl (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.5/126.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting Levenshtein\n", " Downloading Levenshtein-0.20.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (175 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.5/175.5 kB\u001b[0m \u001b[31m34.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting plac\n", " Downloading plac-1.3.5-py2.py3-none-any.whl (22 kB)\n", "Requirement already satisfied: termcolor in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (2.2.0)\n", "Collecting loguru\n", " Downloading loguru-0.6.0-py3-none-any.whl (58 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pathtools\n", " Downloading pathtools-0.1.2.tar.gz (11 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (5.9.4)\n", "Collecting sentry-sdk>=1.0.0\n", " Downloading sentry_sdk-1.17.0-py2.py3-none-any.whl (189 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m189.1/189.1 kB\u001b[0m \u001b[31m41.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting GitPython!=3.1.29,>=1.0.0\n", " Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 kB\u001b[0m \u001b[31m41.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting setproctitle\n", " Downloading setproctitle-1.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n", "Collecting docker-pycreds>=0.4.0\n", " Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n", "Requirement already satisfied: entrypoints in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (0.4)\n", "Requirement already satisfied: jsonschema>=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (3.2.0)\n", "Requirement already satisfied: toolz in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (0.12.0)\n", "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.71->boto3->nemo_toolkit[all]) (1.26.8)\n", "Requirement already satisfied: pycparser in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from cffi>=1.0->soundfile->nemo_toolkit[all]) (2.21)\n", "Requirement already satisfied: itsdangerous>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from Flask>=0.8->flask-restful->nemo_toolkit[all]) (2.1.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.3.3)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (6.0.4)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.3.1)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.8.2)\n", "Collecting gitdb<5,>=4.0.1\n", " Using cached gitdb-4.0.10-py3-none-any.whl (62 kB)\n", "Collecting cachetools<6.0,>=2.0.0\n", " Downloading cachetools-5.3.0-py3-none-any.whl (9.3 kB)\n", "Collecting pyasn1-modules>=0.2.1\n", " Downloading pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m155.3/155.3 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: rsa<5,>=3.1.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (4.7.2)\n", "Collecting requests-oauthlib>=0.7.0\n", " Downloading requests_oauthlib-1.3.1-py2.py3-none-any.whl (23 kB)\n", "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from importlib-metadata>=4.4->sphinx->nemo_toolkit[all]) (3.11.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.1.6)\n", "Requirement already satisfied: comm>=0.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.1.2)\n", "Requirement already satisfied: jupyter-client>=6.1.12 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (7.4.8)\n", "Requirement already satisfied: tornado>=6.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (6.2)\n", "Requirement already satisfied: pyzmq>=17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (24.0.1)\n", "Requirement already satisfied: nest-asyncio in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.5.5)\n", "Requirement already satisfied: debugpy>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.6.5)\n", "Requirement already satisfied: jedi>=0.16 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (0.18.2)\n", "Requirement already satisfied: pexpect>4.3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (4.8.0)\n", "Requirement already satisfied: backcall in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n", "Requirement already satisfied: pickleshare in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (0.7.5)\n", "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (3.0.36)\n", "Collecting mdurl~=0.1\n", " Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n", "Collecting linkify-it-py<3,>=1\n", " Downloading linkify_it_py-2.0.0-py3-none-any.whl (19 kB)\n", "Requirement already satisfied: typer[all]>=0.2.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.4.2)\n", "Collecting latexcodec>=1.0.4\n", " Downloading latexcodec-2.0.1-py2.py3-none-any.whl (18 kB)\n", "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (2022.12.7)\n", "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (3.4)\n", "Collecting urllib3<1.27,>=1.25.4\n", " Using cached urllib3-1.26.15-py2.py3-none-any.whl (140 kB)\n", "Requirement already satisfied: mpmath>=0.19 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sympy>=1.1->pyannote.metrics->nemo_toolkit[all]) (1.2.1)\n", "Requirement already satisfied: soupsieve>1.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from beautifulsoup4->gdown->nemo_toolkit[all]) (2.3.2.post1)\n", "Collecting starlette<0.27.0,>=0.26.1\n", " Downloading starlette-0.26.1-py3-none-any.whl (66 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.9/66.9 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: sniffio in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from httpx->gradio->nemo_toolkit[all]) (1.3.0)\n", "Collecting rfc3986[idna2008]<2,>=1.3\n", " Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n", "Collecting httpcore<0.17.0,>=0.15.0\n", " Downloading httpcore-0.16.3-py3-none-any.whl (69 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (1.7.1)\n", "Collecting h11>=0.8\n", " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting smmap<6,>=3.0.1\n", " Using cached smmap-5.0.0-py3-none-any.whl (24 kB)\n", "Requirement already satisfied: anyio<5.0,>=3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from httpcore<0.17.0,>=0.15.0->httpx->gradio->nemo_toolkit[all]) (3.6.2)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (0.8.3)\n", "Requirement already satisfied: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jsonschema>=3.0->altair>=4.2.0->gradio->nemo_toolkit[all]) (0.19.3)\n", "Requirement already satisfied: jupyter-core>=4.9.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (5.1.3)\n", "Collecting uc-micro-py\n", " Downloading uc_micro_py-1.0.1-py3-none-any.whl (6.2 kB)\n", "Requirement already satisfied: ptyprocess>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets->nemo_toolkit[all]) (0.7.0)\n", "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (0.4.8)\n", "Collecting oauthlib>=3.0.0\n", " Downloading oauthlib-3.2.2-py3-none-any.whl (151 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m151.7/151.7 kB\u001b[0m \u001b[31m29.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: shellingham<2.0.0,>=1.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (1.5.0.post1)\n", "Requirement already satisfied: platformdirs>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jupyter-core>=4.9.2->jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (2.6.2)\n", "Building wheels for collected packages: progress, sacremoses, youtokentome, fasttext, jieba, kaldi-python-io, kaldiio, nemo_toolkit, sentence-transformers, audioread, distance, ipadic, cdifflib, ffmpy, pathtools\n", " Building wheel for progress (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for progress: filename=progress-1.6-py3-none-any.whl size=9611 sha256=d2f329da8f7071861ada35ee5969cd268016dc90a5435fa848b40f8a823779bc\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/4c/9b/0a/a78ff56725af3ef70792f9ed0f8dbbc4c0315edc62cbc4a6b8\n", " Building wheel for sacremoses (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895241 sha256=b3725b02e121e99b6bb2ea2bd1ab443489c1e6589357b74f1414105a52bf95e1\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/12/1c/3d/46cf06718d63a32ff798a89594b61e7f345ab6b36d909ce033\n", " Building wheel for youtokentome (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for youtokentome: filename=youtokentome-1.0.6-cp39-cp39-linux_x86_64.whl size=161353 sha256=3ad12f7936953748a92c3fdb0e3cd7bca11fc9606313928da4ddb8f5d4cd4f52\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/89/99/eb/b72f9c470f14438147d8aa1ff91f9f5191b5e5d825c4b0a12c\n", " Building wheel for fasttext (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for fasttext: filename=fasttext-0.9.2-cp39-cp39-linux_x86_64.whl size=287117 sha256=c5e7981bdd992be2cc8a3ec51a457f562fc58bfcd8535e6e13bb7134e3e1df13\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/64/57/bc/1741406019061d5664914b070bd3e71f6244648732bc96109e\n", " Building wheel for jieba (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=20c5d4d8ce6947b4020a4cd387707bc9643f19d34557f4ceaa4cd2afa52c7846\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/7d/74/cf/08c94db4b784e2c1ef675a600b7b5b281fd25240dcb954ee7e\n", " Building wheel for kaldi-python-io (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for kaldi-python-io: filename=kaldi_python_io-1.2.2-py3-none-any.whl size=8953 sha256=94fe342683586880c9f770034306383e15b5873ce3e54686301cdfd12c3e2878\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/bb/47/c2/801e79937d1342a100b4f2b7d05dcae68d978c2ac22c06ab8e\n", " Building wheel for kaldiio (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for kaldiio: filename=kaldiio-2.17.2-py3-none-any.whl size=24447 sha256=9a11c9f7f31d49f3dde10886ed9eae9400643e65b0e1217196e4fe3269165154\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/5d/8b/4e/17aac05e86a04db23f508115dc5aa24a6077fa5357eaee0d78\n", " Building wheel for nemo_toolkit (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for nemo_toolkit: filename=nemo_toolkit-1.17.0rc0-py3-none-any.whl size=2278167 sha256=2ea4ae407f97145be4c2cee4973010bc9ec65cd8be1563c34f7b1d22333f7ff7\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-e137gpuu/wheels/af/9d/ef/62d10ba057609b5d066b4fd01b97b207270610f648fc0e1812\n", " Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125925 sha256=16502c36f868bb7b5f70d90acf8daa24ce4bac16ab99e37746c3fedd5607d82f\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/71/67/06/162a3760c40d74dd40bc855d527008d26341c2b0ecf3e8e11f\n", " Building wheel for audioread (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for audioread: filename=audioread-3.0.0-py3-none-any.whl size=23704 sha256=2fa786d3683df516349879811f997dc4bff78d72ea3a7bb3cfaac6cfed98591f\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/e4/76/a4/cfb55573167a1f5bde7d7a348e95e509c64b2c3e8f921932c3\n", " Building wheel for distance (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for distance: filename=Distance-0.1.3-py3-none-any.whl size=16257 sha256=1319a507a1c7320e867262a4751a252e71203652a5aeaeee97bca8f6c1aaf85c\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/fb/b3/aa/04241cced6d1722b132273b1d6aafba317887ec004f48b853a\n", " Building wheel for ipadic (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556703 sha256=1da37ecc39259a1674a30ebb19f8f4aab9c6f3a25bcd1b4aa1daa743fba05d74\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/a0/9e/13/a63ab80684c3bd6305fe2bded26ac5f2c72ee1cb07a747994d\n", " Building wheel for cdifflib (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for cdifflib: filename=cdifflib-1.2.6-cp39-cp39-linux_x86_64.whl size=12287 sha256=1fddf5dc4a889ac006739d1531be388d123d1da0327b2a117dd4b74f7ede99da\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/ab/97/fb/48038e1ad28b2447597595cd197785fbae1757f3e3a93d9e27\n", " Building wheel for ffmpy (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for ffmpy: filename=ffmpy-0.3.0-py3-none-any.whl size=4693 sha256=b8a1abac96a30004123bc72d626ddb844c78a7f89d5ada4aef1f206de53a48a2\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/91/e2/96/f676aa08bfd789328c6576cd0f1fde4a3d686703bb0c247697\n", " Building wheel for pathtools (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=e83fc3bc7930e83ac57289ecf8e200ad909a7657894f26e9d52c7425bae10444\n", " Stored in directory: /home/ec2-user/.cache/pip/wheels/b7/0a/67/ada2a22079218c75a88361c0782855cc72aebc4d18d0289d05\n", "Successfully built progress sacremoses youtokentome fasttext jieba kaldi-python-io kaldiio nemo_toolkit sentence-transformers audioread distance ipadic cdifflib ffmpy pathtools\n", "Installing collected packages: tensorboard-plugin-wit, sentencepiece, rfc3986, pydub, progress, plac, pathtools, parameterized, pangu, opencc, mecab-python3, jieba, ipadic, ijson, ffmpy, faiss-cpu, distance, braceexpand, websockets, webdataset, urllib3, uc-micro-py, tensorboard-data-server, soxr, smmap, setuptools, setproctitle, semantic-version, rapidfuzz, python-multipart, pytest-runner, pypinyin, pynini, pyasn1-modules, portalocker, orjson, omegaconf, oauthlib, mdurl, lxml, loguru, lazy-loader, latexcodec, kaldiio, kaldi-python-io, h11, grpcio, ftfy, einops, editdistance, docker-pycreds, click, cdifflib, cachetools, audioread, attrdict, aiofiles, absl-py, youtokentome, uvicorn, torchmetrics, starlette, soundfile, sentry-sdk, sacremoses, sacrebleu, pypinyin-dict, pybtex, pyannote.core, marshmallow, markdown-it-py, markdown, linkify-it-py, lightning-utilities, Levenshtein, kornia, jiwer, inflect, hydra-core, httpcore, google-auth, gitdb, fasttext, black, texterrors, requests-oauthlib, pybtex-docutils, mdit-py-plugins, httpx, GitPython, g2p-en, fastapi, altair, wandb, sphinxcontrib-bibtex, pytorch-lightning, pyannote.database, librosa, gradio, google-auth-oauthlib, gdown, tensorboard, sentence-transformers, pyannote.metrics, nemo-text-processing, nemo_toolkit\n", " Attempting uninstall: urllib3\n", " Found existing installation: urllib3 1.26.8\n", " Uninstalling urllib3-1.26.8:\n", " Successfully uninstalled urllib3-1.26.8\n", " Attempting uninstall: setuptools\n", " Found existing installation: setuptools 65.6.3\n", " Uninstalling setuptools-65.6.3:\n", " Successfully uninstalled setuptools-65.6.3\n", " Attempting uninstall: omegaconf\n", " Found existing installation: omegaconf 2.3.0\n", " Uninstalling omegaconf-2.3.0:\n", " Successfully uninstalled omegaconf-2.3.0\n", " Attempting uninstall: click\n", " Found existing installation: click 8.1.3\n", " Uninstalling click-8.1.3:\n", " Successfully uninstalled click-8.1.3\n", " Attempting uninstall: black\n", " Found existing installation: black 22.10.0\n", " Uninstalling black-22.10.0:\n", " Successfully uninstalled black-22.10.0\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "python-lsp-black 1.2.1 requires black>=22.3.0, but you have black 19.10b0 which is incompatible.\n", "jupyterlab-server 2.18.0 requires jsonschema>=4.17.3, but you have jsonschema 3.2.0 which is incompatible.\n", "distributed 2022.11.0 requires tornado<6.2,>=6.0.3, but you have tornado 6.2 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0mSuccessfully installed GitPython-3.1.31 Levenshtein-0.20.9 absl-py-1.4.0 aiofiles-23.1.0 altair-4.2.2 attrdict-2.0.1 audioread-3.0.0 black-19.10b0 braceexpand-0.1.7 cachetools-5.3.0 cdifflib-1.2.6 click-8.0.2 distance-0.1.3 docker-pycreds-0.4.0 editdistance-0.6.2 einops-0.6.0 faiss-cpu-1.7.3 fastapi-0.95.0 fasttext-0.9.2 ffmpy-0.3.0 ftfy-6.1.1 g2p-en-2.1.0 gdown-4.6.4 gitdb-4.0.10 google-auth-2.16.2 google-auth-oauthlib-0.4.6 gradio-3.23.0 grpcio-1.51.3 h11-0.14.0 httpcore-0.16.3 httpx-0.23.3 hydra-core-1.2.0 ijson-3.2.0.post0 inflect-6.0.2 ipadic-1.0.0 jieba-0.42.1 jiwer-2.5.2 kaldi-python-io-1.2.2 kaldiio-2.17.2 kornia-0.6.10 latexcodec-2.0.1 lazy-loader-0.2 librosa-0.10.0.post2 lightning-utilities-0.8.0 linkify-it-py-2.0.0 loguru-0.6.0 lxml-4.9.2 markdown-3.4.1 markdown-it-py-2.2.0 marshmallow-3.19.0 mdit-py-plugins-0.3.3 mdurl-0.1.2 mecab-python3-1.0.5 nemo-text-processing-0.1.7rc0 nemo_toolkit-1.17.0rc0 oauthlib-3.2.2 omegaconf-2.2.3 opencc-1.1.6 orjson-3.8.8 pangu-4.0.6.1 parameterized-0.8.1 pathtools-0.1.2 plac-1.3.5 portalocker-2.7.0 progress-1.6 pyannote.core-5.0.0 pyannote.database-4.1.3 pyannote.metrics-3.2.1 pyasn1-modules-0.2.8 pybtex-0.24.0 pybtex-docutils-1.0.2 pydub-0.25.1 pynini-2.1.5 pypinyin-0.48.0 pypinyin-dict-0.5.0 pytest-runner-6.0.0 python-multipart-0.0.6 pytorch-lightning-1.9.4 rapidfuzz-2.13.7 requests-oauthlib-1.3.1 rfc3986-1.5.0 sacrebleu-2.3.1 sacremoses-0.0.53 semantic-version-2.10.0 sentence-transformers-2.2.2 sentencepiece-0.1.97 sentry-sdk-1.17.0 setproctitle-1.3.2 setuptools-65.5.1 smmap-5.0.0 soundfile-0.12.1 soxr-0.3.4 sphinxcontrib-bibtex-2.5.0 starlette-0.26.1 tensorboard-2.12.0 tensorboard-data-server-0.7.0 tensorboard-plugin-wit-1.8.1 texterrors-0.4.4 torchmetrics-0.11.4 uc-micro-py-1.0.1 urllib3-1.26.15 uvicorn-0.21.1 wandb-0.14.0 webdataset-0.1.62 websockets-10.4 youtokentome-1.0.6\n" ] } ], "source": [ "import sys\n", "import IPython\n", "\n", "if install_needed:\n", " print(\"installing deps and restarting kernel\")\n", " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install -U smdebug sagemaker-experiments\n", " !{sys.executable} -m pip install -U sagemaker\n", " !{sys.executable} -m pip install -U datasets transformers\n", " !{sys.executable} -m pip install -U wget omegaconf text-unidecode sox\n", " \n", " ## Install NeMo\n", " !sudo yum install sox -y\n", " !sudo yum install libsndfile -y\n", " !pip install --upgrade --force-reinstall llvmlite\n", " BRANCH = 'main'\n", " !{sys.executable} -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[all]\n", " \n", " IPython.Application.instance().kernel.do_shutdown(True)" ] }, { "cell_type": "markdown", "id": "9d235cc4-356c-43e0-99ca-fea913797c99", "metadata": { "tags": [] }, "source": [ "## 1. Set roles" ] }, { "cell_type": "code", "execution_count": 2, "id": "1f3a2019-2337-4b48-9bd8-f577ac13daed", "metadata": { "tags": [] }, "outputs": [], "source": [ "from sagemaker import get_execution_role" ] }, { "cell_type": "code", "execution_count": 3, "id": "3be71d18-b4b7-4196-abf1-1922f3013c4e", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "SageMaker Execution Role Name: AmazonSageMaker-ExecutionRole-20221206T163436\n" ] } ], "source": [ "strSageMakerRoleName = get_execution_role().rsplit('/', 1)[-1]\n", "print (f\"SageMaker Execution Role Name: {strSageMakerRoleName}\")" ] }, { "cell_type": "markdown", "id": "8acda191-5a24-4904-b637-92e9bfd580e2", "metadata": { "tags": [] }, "source": [ "## 1.1 Attach IAM polich to sagemaker execution role (with console)\n", "> **EC2ContainerRegistry**: \"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess\"
\n", "> **S3**: \"arn:aws:iam::aws:policy/AmazonS3FullAccess\"" ] }, { "cell_type": "markdown", "id": "87fe7a4b-03e1-4422-8bd5-62d30e0a419f", "metadata": {}, "source": [ "## 2. Set default parameters" ] }, { "cell_type": "code", "execution_count": 4, "id": "e5e8ad58-6a1e-41b4-88e5-aec263ef935e", "metadata": { "tags": [] }, "outputs": [], "source": [ "import boto3\n", "import sagemaker" ] }, { "cell_type": "markdown", "id": "92d651c6-93ad-463b-905a-d9978bd748a7", "metadata": {}, "source": [ "### Bucket / Prefix 설정" ] }, { "cell_type": "code", "execution_count": 5, "id": "659a6734-542a-4f79-b4bb-726408709308", "metadata": { "tags": [] }, "outputs": [], "source": [ "strRegionName = boto3.Session().region_name\n", "strAccountId = boto3.client(\"sts\").get_caller_identity().get(\"Account\")\n", "bucket_name = 'sm-nemo-ramp' # <-- 사용할 bucket 명을 추가해 주세요. ex) sagemaker-us-east-1-123456789123, sm-nemo-bucket\n", "prefix = 'nemo-asr' ## <-- 작업할 prefix 명을 추가해 주세요. ex) nemo-test, nemo-asr" ] }, { "cell_type": "markdown", "id": "c82a79d6-71b2-48fb-8c80-41a9549b36c5", "metadata": {}, "source": [ "## 3. Create training custom docker image" ] }, { "cell_type": "markdown", "id": "84b2409b-8c8f-420f-bf7d-0ca95880b92f", "metadata": {}, "source": [ "* docker build" ] }, { "cell_type": "code", "execution_count": null, "id": "e2c97bbe-918a-454e-a444-0a03bc5c48b1", "metadata": { "tags": [] }, "outputs": [], "source": [ "!pygmentize custom-docker/Dockerfile" ] }, { "cell_type": "markdown", "id": "9816ab13-8047-4f06-9fdc-97999a974bd1", "metadata": {}, "source": [ "* Base 이미지의 region, account-id 확인 후 아래 파라미터 입력" ] }, { "cell_type": "markdown", "id": "c8a73b5e-1775-41f2-bf79-174cebab1778", "metadata": {}, "source": [ "### 1) AWS CLI 를 이용한 방식" ] }, { "cell_type": "code", "execution_count": null, "id": "eb1c8b5e-f0d6-4194-a8aa-6328b68c1280", "metadata": { "tags": [] }, "outputs": [], "source": [ "%%bash\n", "strRepositoryName=\"nemo-test-training\" ## <-- 원하는 docker repostory 이름을 추가\n", "strDockerDir=\"./custom-docker/\"\n", "strTag=\"latest\"\n", "\n", "cd ${strDockerDir}\n", "echo $(pwd)\n", "container_name=${strRepositoryName}\n", "\n", "account=$(aws sts get-caller-identity --query Account --output text)\n", "\n", "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n", "region=$(aws configure get region)\n", "# region=${region:-us-west-2}\n", "\n", "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${container_name}:${strTag}\"\n", "\n", "# If the repository doesn't exist in ECR, create it.\n", "aws ecr describe-repositories --repository-names \"${container_name}\" > /dev/null 2>&1\n", "if [ $? -ne 0 ]\n", "then\n", " aws ecr create-repository --repository-name \"${container_name}\" > /dev/null\n", "fi\n", "\n", "# # Get the login command from ECR and execute it directly\n", "# $(aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin \"763104351884.dkr.ecr.us-west-2.amazonaws.com\")\n", "\n", "# Build the docker image locally with the image name and then push it to ECR\n", "# with the full name.\n", "docker build -f Dockerfile -t ${fullname} .\n", "# docker tag ${container_name} ${fullname}\n", "\n", "# Get the login command from ECR and execute it directly\n", "$(aws ecr get-login --region ${region} --no-include-email)\n", "docker push ${fullname}\n", "echo ${fullname}" ] }, { "cell_type": "code", "execution_count": 6, "id": "0a80bf77-8063-476d-b54c-8ddf82f88994", "metadata": { "tags": [] }, "outputs": [], "source": [ "strEcrRepositoryUri = '419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-training'#' ' ## <-- 생성된 ECR의 URI를 넣어주세요.. ex) 123456789123.dkr.ecr.us-west-2.amazonaws.com/nemo-test:latest" ] }, { "cell_type": "markdown", "id": "f59e141b-07af-421e-a05d-bc6991dce9df", "metadata": { "tags": [] }, "source": [ "### 2) AWS BOTO3 SDK를 이용한 방식" ] }, { "cell_type": "code", "execution_count": null, "id": "6bf0f03e-a0fe-44af-ac74-c9a67fd1bbc0", "metadata": { "tags": [] }, "outputs": [], "source": [ "from utils.ecr import ecr_handler\n", "ecr = ecr_handler()" ] }, { "cell_type": "code", "execution_count": null, "id": "1c68ef98-9669-4ea8-8e39-7717d0ffa831", "metadata": { "tags": [] }, "outputs": [], "source": [ "strRepositoryName=\"nemo-test-training\" ## <-- 원하는 docker repostory 이름을 추가\n", "strRepositoryName = strRepositoryName.lower()\n", "strDockerDir = \"./custom-docker/\"\n", "strDockerFile = \"Dockerfile\"\n", "strTag = \"latest\"" ] }, { "cell_type": "code", "execution_count": null, "id": "a4bf0924-52d8-457c-9edb-aff333850c1a", "metadata": { "tags": [] }, "outputs": [], "source": [ "ecr.build_docker(strDockerDir, strDockerFile, strRepositoryName, strRegionName=\"us-west-2\", strAccountId=\"763104351884\")" ] }, { "cell_type": "markdown", "id": "9e1d1a7a-6677-462a-87b3-6ba628884276", "metadata": {}, "source": [ "* Push the image to ECR" ] }, { "cell_type": "code", "execution_count": null, "id": "09fa8f56-7ac2-440d-86f0-6e6a07dc71a4", "metadata": { "tags": [] }, "outputs": [], "source": [ "strEcrRepositoryUri = ecr.register_image_to_ecr(strRegionName, strAccountId, strRepositoryName, strTag)" ] }, { "cell_type": "markdown", "id": "e1b0385b-d1a6-4281-87fa-261cbf5acbbd", "metadata": {}, "source": [ "* Save image-uri to parameter store" ] }, { "cell_type": "markdown", "id": "2fbcd0ad-384a-4c24-922d-a17213ec1dba", "metadata": {}, "source": [ "## 4. Create inference custom docker image" ] }, { "cell_type": "markdown", "id": "9a80f82b-edcd-4f86-8211-f79fbd5d4adf", "metadata": {}, "source": [ "* docker build" ] }, { "cell_type": "code", "execution_count": null, "id": "886a0a4b-0601-4674-a575-88c5051989ab", "metadata": { "tags": [] }, "outputs": [], "source": [ "!pygmentize custom-docker/Dockerfile.inf" ] }, { "cell_type": "markdown", "id": "2a45013c-144a-4918-b154-e9cd7425506a", "metadata": {}, "source": [ "### 1) AWS CLI 를 이용한 방식" ] }, { "cell_type": "code", "execution_count": null, "id": "1eed522b-5d65-4a69-9502-298504d5b7c5", "metadata": { "tags": [] }, "outputs": [], "source": [ "%%bash\n", "strRepositoryName=\"nemo-test-inference\"\n", "strDockerDir=\"./custom-docker/\"\n", "strTag=\"latest\"\n", "\n", "cd ${strDockerDir}\n", "echo $(pwd)\n", "container_name=${strRepositoryName}\n", "\n", "account=$(aws sts get-caller-identity --query Account --output text)\n", "\n", "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n", "region=$(aws configure get region)\n", "# region=${region:-us-west-2}\n", "\n", "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${container_name}:${strTag}\"\n", "\n", "# If the repository doesn't exist in ECR, create it.\n", "aws ecr describe-repositories --repository-names \"${container_name}\" > /dev/null 2>&1\n", "if [ $? -ne 0 ]\n", "then\n", " aws ecr create-repository --repository-name \"${container_name}\" > /dev/null\n", "fi\n", "\n", "# # Get the login command from ECR and execute it directly\n", "# $(aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin \"763104351884.dkr.ecr.us-west-2.amazonaws.com\")\n", "\n", "# Build the docker image locally with the image name and then push it to ECR\n", "# with the full name.\n", "docker build -f Dockerfile.inf -t ${fullname} .\n", "# docker tag ${container_name} ${fullname}\n", "\n", "# Get the login command from ECR and execute it directly\n", "$(aws ecr get-login --region ${region} --no-include-email)\n", "docker push ${fullname}\n", "echo ${fullname}" ] }, { "cell_type": "code", "execution_count": 7, "id": "9766980f-4a8d-48f8-986f-5f75ada29c08", "metadata": { "tags": [] }, "outputs": [], "source": [ "strInfEcrRepositoryUri = '419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-inference' ## <-- 생성된 ECR의 URI를 넣어주세요.. ex) 123456789123.dkr.ecr.us-west-2.amazonaws.com/nemo-test:latest\n" ] }, { "cell_type": "markdown", "id": "e4af3f7e-90a4-4923-b44f-6af2efe8fd8b", "metadata": {}, "source": [ "### 2) AWS BOTO3 SDK를 이용한 방식" ] }, { "cell_type": "code", "execution_count": 33, "id": "1e8bf58e-7ad3-4827-acdc-ec64f90c8b46", "metadata": { "tags": [] }, "outputs": [], "source": [ "from utils.ecr import ecr_handler\n", "ecr = ecr_handler()" ] }, { "cell_type": "code", "execution_count": 34, "id": "897f4ee5-8440-426d-83ff-af5ef558bda8", "metadata": { "tags": [] }, "outputs": [], "source": [ "strInfRepositoryName=\"nemo-test-inference\" ## <-- 원하는 docker repostory 이름을 추가\n", "strInfRepositoryName = strInfRepositoryName.lower()\n", "strDockerFile = \"Dockerfile.inf\"\n", "strDockerDir = \"./custom-docker/\"\n", "strTag = \"latest\"" ] }, { "cell_type": "code", "execution_count": 35, "id": "2e6a9c7a-67dc-4951-8d7b-fa261bdcdb22", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/ec2-user/SageMaker/nemo-on-sagemaker/1.building-component\n", "/home/ec2-user/SageMaker/nemo-on-sagemaker/1.building-component/custom-docker\n", "strDockerFile Dockerfile.inf\n", "aws ecr get-login --region 'us-west-2' --registry-ids '763104351884' --no-include-email\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING! Using --password via the CLI is insecure. Use --password-stdin.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Login Succeeded\n", "\n", "Sending build context to Docker daemon 10.75kB\n", "\n", "Step 1/4 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.13.1-gpu-py39\n", " ---> 58538dc47aa0\n", "Step 2/4 : RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -U sagemaker nvgpu hydra-core librosa sentencepiece youtokentome inflect pyannote.audio && pip install --no-cache-dir -U braceexpand webdataset editdistance jiwer jsonlines && pip install --no-cache-dir pytorch-lightning==1.9.4 && pip install --no-cache-dir git+https://github.com/huggingface/transformers && pip install --no-cache-dir git+https://github.com/NVIDIA/NeMo.git@main\n", " ---> Using cache\n", " ---> 9167df4989d0\n", "Step 3/4 : COPY list_gpus.py /opt/conda/lib/python3.9/site-packages/nvgpu/list_gpus.py\n", " ---> Using cache\n", " ---> 72636f7b8709\n", "Step 4/4 : WORKDIR /\n", " ---> Using cache\n", " ---> ea426d21de1c\n", "Successfully built ea426d21de1c\n", "Successfully tagged nemo-test-inference:latest\n", "\n", "/home/ec2-user/SageMaker/nemo-on-sagemaker/1.building-component\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n", "Configure a credential helper to remove this warning. See\n", "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n", "\n" ] } ], "source": [ "ecr.build_docker(strDockerDir, strDockerFile, strInfRepositoryName, strRegionName=\"us-west-2\", strAccountId=\"763104351884\")" ] }, { "cell_type": "code", "execution_count": 36, "id": "d3a29604-f58b-4106-a23d-6d36c30727f4", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "== REGISTER AN IMAGE TO ECR ==\n", " processing_repository_uri: 419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-inference:latest\n", "aws ecr get-login --region 'us-east-1' --registry-ids '419974056037' --no-include-email\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING! Using --password via the CLI is insecure. Use --password-stdin.\n", "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n", "Configure a credential helper to remove this warning. See\n", "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Login Succeeded\n", "\n", "aws ecr create-repository --repository-name 'nemo-test-inference'\n", "docker tag 'nemo-test-inference:latest' '419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-inference:latest'\n", "docker push '419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-inference:latest'\n", "== REGISTER AN IMAGE TO ECR ==\n", "==============================\n" ] } ], "source": [ "strInfEcrRepositoryUri = ecr.register_image_to_ecr(strRegionName, strAccountId, strInfRepositoryName, strTag)" ] }, { "cell_type": "markdown", "id": "66618f51-efdf-4336-8712-acf2201f5aab", "metadata": {}, "source": [ "## 5. Download & Upload dataset" ] }, { "cell_type": "code", "execution_count": 8, "id": "693ac722-d6d3-4ae0-8787-27dd7a3a11cc", "metadata": { "tags": [] }, "outputs": [], "source": [ "import os\n", "import wget" ] }, { "cell_type": "code", "execution_count": 9, "id": "b3b1eb5b-ba2f-40e1-824c-846374d53bad", "metadata": { "tags": [] }, "outputs": [], "source": [ "data_dir = \"./data\"" ] }, { "cell_type": "code", "execution_count": 10, "id": "406951c5-4597-49f4-9ced-db4aeef0512d", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "******\n", "Dataset downloaded at: ./data/an4_sphere.tar.gz\n" ] } ], "source": [ "print(\"******\")\n", "os.makedirs(data_dir, exist_ok=True)\n", "if not os.path.exists(data_dir + '/an4_sphere.tar.gz'):\n", " an4_url = 'https://dldata-public.s3.us-east-2.amazonaws.com/an4_sphere.tar.gz'\n", " an4_path = wget.download(an4_url, data_dir)\n", " print(f\"Dataset downloaded at: {an4_path}\")\n", "else:\n", " print(\"Tarfile already exists.\")\n", " an4_path = data_dir + '/an4_sphere.tar.gz'" ] }, { "cell_type": "markdown", "id": "3ac1ad94-a3d0-49a2-b16a-35887875ed54", "metadata": {}, "source": [ "* updoad data to s3" ] }, { "cell_type": "markdown", "id": "1c5f28dd-98a4-45f8-87f2-21eb1f942be7", "metadata": {}, "source": [ "### 1) AWS CLI 를 이용한 방식" ] }, { "cell_type": "code", "execution_count": 11, "id": "56651221-771b-4350-8b21-bd316235ae44", "metadata": { "tags": [] }, "outputs": [], "source": [ "!aws s3 sync $data_dir s3://$bucket_name/$prefix/data --quiet" ] }, { "cell_type": "markdown", "id": "651df427-7e23-4861-906f-4f1536c4cc90", "metadata": {}, "source": [ "### 2) AWS BOTO3 SDK를 이용한 방식" ] }, { "cell_type": "code", "execution_count": null, "id": "d433d60e-f53f-403e-bf99-1bc52e6e5456", "metadata": { "tags": [] }, "outputs": [], "source": [ "from utils.s3 import s3_handler" ] }, { "cell_type": "code", "execution_count": null, "id": "35d1201e-0f10-48c8-9a20-83f4ff20efde", "metadata": { "tags": [] }, "outputs": [], "source": [ "s3 = s3_handler()" ] }, { "cell_type": "code", "execution_count": null, "id": "95e9f4ec-f3ad-4a63-aa4a-425615aea2c8", "metadata": { "tags": [] }, "outputs": [], "source": [ "source_dir, target_bucket, target_dir = data_dir, pm.get_params(key=prefix+\"-BUCKET\"), prefix+\"/data\"\n", "s3.upload_dir(source_dir, target_bucket, target_dir)" ] }, { "cell_type": "markdown", "id": "a56442a3-3241-4d78-92d6-67cb053f8b04", "metadata": {}, "source": [ "## 6. Upload Pretrained model" ] }, { "cell_type": "code", "execution_count": 12, "id": "4c7176b3-28d9-49f7-9a6b-bc36128022a5", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: pretrained/CTC.nemo to s3://sm-nemo-ramp/nemo-asr/pretrained/CTC.nemo\n" ] } ], "source": [ "pretrained = os.getcwd() + '/pretrained/CTC.nemo'\n", "\n", "pretrained_s3uri = os.path.join(\n", " \"s3://{}\".format(bucket_name),\n", " prefix,\n", " \"pretrained\",\n", ")\n", "\n", "!aws s3 sync pretrained $pretrained_s3uri" ] }, { "cell_type": "markdown", "id": "6d5f0673-538d-4e72-94ca-663177ff12ae", "metadata": {}, "source": [ "## 7. CodeCommit 생성\n", "- Attach IAM polich to sagemaker execution role (with console)\n", "> **CodeCommit**: \"arn:aws:iam::aws:policy/AWSCodeCommitFullAccess\"
\n", "> **SecretsManager**: \"arn:aws:iam::aws:policy/SecretsManagerReadWrite\"
" ] }, { "cell_type": "markdown", "id": "d9fca091-a616-41c1-a522-7327720ec74e", "metadata": {}, "source": [ "### 5.1 CodeCommit 관련 Credentials 생성 및 Secret Manager에 저장하기\n", "- CodeCommit Credentials" ] }, { "cell_type": "code", "execution_count": 13, "id": "3d570c6e-58f8-4749-be70-0687ee606a3a", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "'codecommit-cred-dongjin'" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "user_name = 'dongjin' ## ==> IAM에서 사용자 아이디 확인합니다.\n", "codecommit_cred = 'codecommit-cred-'+user_name\n", "codecommit_cred" ] }, { "cell_type": "code", "execution_count": 14, "id": "aa00bbd2-bf62-46d8-9e20-75f2a0f8910b", "metadata": { "tags": [] }, "outputs": [], "source": [ "iam_client = boto3.client('iam')" ] }, { "cell_type": "code", "execution_count": 15, "id": "6435b225-db6a-4b71-8fc2-4e3bc52b788e", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ServiceUserName : dongjin-at-419974056037 \n", "ServicePassword : wtLv/fP4ESjBDnyW5xgqFPGR0dMTIyK5/8gK6IS1Zsg=\n" ] } ], "source": [ "try:\n", " response = iam_client.list_service_specific_credentials(\n", " UserName=user_name,\n", " ServiceName='codecommit.amazonaws.com'\n", " )\n", " if len(response['ServiceSpecificCredentials']) > 0:\n", " response = iam_client.delete_service_specific_credential(\n", " UserName=user_name,\n", " ServiceSpecificCredentialId=response['ServiceSpecificCredentials'][-1]['ServiceSpecificCredentialId']\n", " )\n", "except:\n", " print(\"Create new codecommit crendentials\")\n", " pass\n", "finally:\n", " response = iam_client.create_service_specific_credential(\n", " UserName=user_name,\n", " ServiceName='codecommit.amazonaws.com'\n", " )\n", " ServiceUserName = response['ServiceSpecificCredential']['ServiceUserName']\n", " ServicePassword = response['ServiceSpecificCredential']['ServicePassword']\n", "print(f\"ServiceUserName : {ServiceUserName} \\nServicePassword : {ServicePassword}\")" ] }, { "cell_type": "code", "execution_count": 16, "id": "745a1773-919a-4074-acc5-28ca6a546a08", "metadata": { "tags": [] }, "outputs": [], "source": [ "code_repository_name = 'nemo-code' ## ==> 사용할 code repository 폴더 명을 넣습니다. ex) model_code\n", "local_code_dir = './code' ## ==> 생성한 local의 code repository 폴더 명을 넣습니다. ex) code" ] }, { "cell_type": "code", "execution_count": 17, "id": "9f19a2b9-6e38-4e0e-ac44-b0b4de22a460", "metadata": { "tags": [] }, "outputs": [], "source": [ "codecommit = boto3.client('codecommit')\n", "\n", "try:\n", " response = codecommit.create_repository(\n", " repositoryName=code_repository_name,\n", " repositoryDescription='Data Scientists share their training code using this Repository'\n", " )\n", "except:\n", " \n", " print(\"Repository already exists\")\n", " response = codecommit.get_repository(\n", " repositoryName=code_repository_name\n", " )" ] }, { "cell_type": "code", "execution_count": 18, "id": "ac4c1e77-f894-4e7c-81ab-9054c7515d61", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "'https://git-codecommit.us-east-1.amazonaws.com/v1/repos/nemo-code'" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "codecommit_repo = response['repositoryMetadata']['cloneUrlHttp']\n", "codecommit_repo" ] }, { "cell_type": "code", "execution_count": null, "id": "5e769023-46fc-482a-948f-27f556bf2805", "metadata": { "tags": [] }, "outputs": [], "source": [ "!rm -rf .git/" ] }, { "cell_type": "code", "execution_count": 23, "id": "7ab59dbf-196b-4a32-bd49-c003342c715d", "metadata": { "collapsed": true, "jupyter": { "outputs_hidden": true }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[33mhint: Using 'master' as the name for the initial branch. This default branch name\u001b[m\n", "\u001b[33mhint: is subject to change. To configure the initial branch name to use in all\u001b[m\n", "\u001b[33mhint: of your new repositories, which will suppress this warning, call:\u001b[m\n", "\u001b[33mhint: \u001b[m\n", "\u001b[33mhint: \tgit config --global init.defaultBranch \u001b[m\n", "\u001b[33mhint: \u001b[m\n", "\u001b[33mhint: Names commonly chosen instead of 'master' are 'main', 'trunk' and\u001b[m\n", "\u001b[33mhint: 'development'. The just-created branch can be renamed via this command:\u001b[m\n", "\u001b[33mhint: \u001b[m\n", "\u001b[33mhint: \tgit branch -m \u001b[m\n", "Initialized empty Git repository in /home/ec2-user/SageMaker/nemo-on-sagemaker/1.building-component/.git/\n", "Switched to a new branch 'main'\n", "[main (root-commit) 3c3cb2a] code-update\n", " Committer: EC2 Default User \n", "Your name and email address were configured automatically based\n", "on your username and hostname. Please check that they are accurate.\n", "You can suppress this message by setting them explicitly:\n", "\n", " git config --global user.name \"Your Name\"\n", " git config --global user.email you@example.com\n", "\n", "After doing this, you may fix the identity used for this commit with:\n", "\n", " git commit --amend --reset-author\n", "\n", " 6 files changed, 1003 insertions(+)\n", " create mode 100644 code/conf/config.yaml\n", " create mode 100644 code/conf/config.yaml.bak\n", " create mode 100644 code/evaluate.py\n", " create mode 100644 code/predictor.py\n", " create mode 100644 code/preprocessing.py\n", " create mode 100644 code/speech_to_text_ctc.py\n", "Enumerating objects: 10, done.\n", "Counting objects: 100% (10/10), done.\n", "Delta compression using up to 8 threads\n", "Compressing objects: 100% (9/9), done.\n", "Writing objects: 100% (10/10), 9.49 KiB | 4.74 MiB/s, done.\n", "Total 10 (delta 1), reused 0 (delta 0), pack-reused 0\n", "remote: Validating objects: 100%\u001b[K\n", "To https://git-codecommit.us-east-1.amazonaws.com/v1/repos/nemo-code\n", " * [new branch] main -> main\n", "branch 'main' set up to track 'repo_codecommit/main'.\n" ] } ], "source": [ "!git init\n", "!git remote add repo_codecommit $codecommit_repo\n", "!git checkout -b main\n", "!git add $local_code_dir\n", "!git commit -m \"code-update\"\n", "!git push --set-upstream repo_codecommit main" ] }, { "cell_type": "markdown", "id": "79d1cc04-a7d2-47a2-bdeb-0357a67aa775", "metadata": {}, "source": [ "## 6. [Optional] AWS Systems Manager Parameter Store 를 이용한 파라미터 저장/활용\n", "- [AWS Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html)\n", "- Attach IAM polich to sagemaker execution role (with console)\n", "> **SSM**: \"arn:aws:iam::aws:policy/AmazonSSMFullAccess\"
" ] }, { "cell_type": "code", "execution_count": 19, "id": "ef82d683-4008-4337-b756-fed86c4d6fc4", "metadata": { "tags": [] }, "outputs": [], "source": [ "from utils.ssm import parameter_store" ] }, { "cell_type": "code", "execution_count": 20, "id": "8cbafaa9-11bb-4207-ac4d-912e02f64709", "metadata": { "tags": [] }, "outputs": [], "source": [ "pm = parameter_store(strRegionName)" ] }, { "cell_type": "code", "execution_count": 26, "id": "0d074104-5c46-489b-982a-4e3801697757", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "'Store suceess'" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pm.put_params(key=\"PREFIX\", value=prefix, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"REGION\"]), value=strRegionName, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"BUCKET\"]), value=bucket_name, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"SAGEMAKER-ROLE-ARN\"]), value=get_execution_role(), overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"ACCOUNT-ID\"]), value=strAccountId, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"IMAGE-URI\"]), value=strEcrRepositoryUri, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"INF-IMAGE-URI\"]), value=strInfEcrRepositoryUri, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"S3-DATA-PATH\"]), value=f\"s3://{bucket_name}/{prefix}/data\", overwrite=True)\n", "#pm.put_params(key=\"-\".join([prefix, \"CODE_REPO\"]), value=codecommit_repo.replace('https://',''), overwrite=True) ## https:// 있을 경우 입력 못함\n", "pm.put_params(key=\"-\".join([prefix, \"CODECOMMIT-USERNAME\"]), value=ServiceUserName, overwrite=True, enc=True)\n", "pm.put_params(key=\"-\".join([prefix, \"CODECOMMIT-PWD\"]), value=ServicePassword, overwrite=True, enc=True)\n", "pm.put_params(key=\"-\".join([prefix, \"PRETRAINED-WEIGHT\"]), value=pretrained_s3uri, overwrite=True)\n", "pm.put_params(key=\"-\".join([prefix, \"RETRAIN\"]), value=False, overwrite=True)" ] }, { "cell_type": "code", "execution_count": 28, "id": "7b3f10d7-7a58-4249-a160-076e4bb19bdc", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PREFIX: nemo-asr\n", "REGION: us-east-1\n", "BUCKET: sm-nemo-ramp\n", "SAGEMAKER-ROLE-ARN: arn:aws:iam::419974056037:role/service-role/AmazonSageMaker-ExecutionRole-20221206T163436\n", "ACCOUNT-ID: 419974056037\n", "IMAGE-URI: 419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-training\n", "INF-IMAGE-URI: 419974056037.dkr.ecr.us-east-1.amazonaws.com/nemo-test-inference\n", "S3-DATA-PATH: s3://sm-nemo-ramp/nemo-asr/data\n", "CODE_REPO: git-codecommit.us-east-1.amazonaws.com/v1/repos/nemo-code\n", "CODECOMMIT-USERNAME: AQICAHixC/mZVJcnQEHIgIK/d13m2pRN5MnNJb7dfKPh/9fZ0QGVuZrBQ7W5B/Cj1oZume/1AAAAdTBzBgkqhkiG9w0BBwagZjBkAgEAMF8GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMtW7mBk1BA6fgi+0VAgEQgDJGMPSMIYkgaFpV+HE3Rbca28eOkuoG5BqDwQwZzdd/KXD16cp+Sx+27D9TqE2ZAXGy7g==\n", "CODECOMMIT-PWD: AQICAHixC/mZVJcnQEHIgIK/d13m2pRN5MnNJb7dfKPh/9fZ0QHUzV+0Wm5yTR1IjSI8FYX4AAAAizCBiAYJKoZIhvcNAQcGoHsweQIBADB0BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGD2gd38cN8IbiI3eQIBEIBHLxltBZUR2+CxagsfFsh2EbMN2NA3s0L+aoh/LWu8hae0KAkN9HSn92gXO6rp0y2602plsH8UGiSM4tEWbXmk2uQWR+bufBA=\n", "PRETRAINED-WEIGHT: s3://sm-nemo-ramp/nemo-asr/pretrained\n", "RETRAIN: False\n" ] } ], "source": [ "print (f'PREFIX: {pm.get_params(key=\"PREFIX\")}')\n", "print (f'REGION: {pm.get_params(key=\"-\".join([prefix, \"REGION\"]))}')\n", "print (f'BUCKET: {pm.get_params(key=\"-\".join([prefix, \"BUCKET\"]))}')\n", "print (f'SAGEMAKER-ROLE-ARN: {pm.get_params(key=\"-\".join([prefix, \"SAGEMAKER-ROLE-ARN\"]))}')\n", "print (f'ACCOUNT-ID: {pm.get_params(key=\"-\".join([prefix, \"ACCOUNT-ID\"]))}')\n", "print (f'IMAGE-URI: {pm.get_params(key=\"-\".join([prefix, \"IMAGE-URI\"]))}')\n", "print (f'INF-IMAGE-URI: {pm.get_params(key=\"-\".join([prefix, \"INF-IMAGE-URI\"]))}')\n", "print (f'S3-DATA-PATH: {pm.get_params(key=\"-\".join([prefix, \"S3-DATA-PATH\"]))}')\n", "print (f'CODE_REPO: {pm.get_params(key=\"-\".join([prefix, \"CODE_REPO\"]))}')\n", "print (f'CODECOMMIT-USERNAME: {pm.get_params(key=\"-\".join([prefix, \"CODECOMMIT-USERNAME\"]), enc=False)}')\n", "print (f'CODECOMMIT-PWD: {pm.get_params(key=\"-\".join([prefix, \"CODECOMMIT-PWD\"]), enc=False)}')\n", "print (f'PRETRAINED-WEIGHT: {pm.get_params(key=\"-\".join([prefix, \"PRETRAINED-WEIGHT\"]))}')\n", "print (f'RETRAIN: {pm.get_params(key=\"-\".join([prefix, \"RETRAIN\"]))}')" ] }, { "cell_type": "code", "execution_count": null, "id": "a8bb59b3-1412-430f-bc75-0f2cea44cd18", "metadata": {}, "outputs": [], "source": [ "https://git-codecommit.us-east-1.amazonaws.com/v1/repos/nemo-code\n", "f'https://git-codecommit.${region}.amazonaws.com/v1/repos/informer2020'" ] } ], "metadata": { "instance_type": "ml.t3.medium", "kernelspec": { "display_name": "conda_pytorch_p39", "language": "python", "name": "conda_pytorch_p39" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.15" }, "vscode": { "interpreter": { "hash": "1f4a227f2552de404565d6626d9115986c409361ad6ef0bf195ad88ccd012b73" } } }, "nbformat": 4, "nbformat_minor": 5 }