{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Train a model locally or remote" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Note: you may need to restart the kernel to use updated packages.\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "%pip uninstall -y ezsmdeploy --quiet\n", "# !pip install --upgrade pip\n", "%pip install ezsmdeploy==2.0.0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Note: you may need to restart the kernel to use updated packages." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Testing foundation model " ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "scrolled": true, "tags": [] }, "outputs": [], "source": [ "# %pip uninstall -y sagemaker\n", "# %pip install --upgrade sagemaker==2.148.0 --no-cache-dir" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "'2.171.0'" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import sagemaker\n", "sagemaker.__version__" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "!pip show ezsmdeploy" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import ezsmdeploy" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "ezonsm = ezsmdeploy.Deploy(model = \"tiiuae/facon-7b\", #\"distilbert-base-uncased-distilled-squad\",\n", " huggingface_model=True,\n", " foundation_model=True,\n", " instance_type='ml.g5.8xlarge'\n", " )" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "tags": [] }, "outputs": [ { "ename": "ModelError", "evalue": "An error occurred (ModelError) when calling the InvokeEndpoint operation: Received client error (400) from primary with message \"{\n \"code\": 400,\n \"type\": \"InternalServerException\",\n \"message\": \"\\u0027gpt_bigcode\\u0027\"\n}\n\". See https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/aws/sagemaker/Endpoints/ezsm-hf-endpoint-g7hxyhngk6pqgridaw6mhq in account 607789152815 for more information.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mModelError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mezonsm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredictor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdef print_hello_world():\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/base_predictor.py:167\u001b[0m, in \u001b[0;36mPredictor.predict\u001b[0;34m(self, data, initial_args, target_model, target_variant, inference_id)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Return the inference from the specified endpoint.\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \n\u001b[1;32m 139\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;124;03m as is.\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 164\u001b[0m request_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_request_args(\n\u001b[1;32m 165\u001b[0m data, initial_args, target_model, target_variant, inference_id\n\u001b[1;32m 166\u001b[0m )\n\u001b[0;32m--> 167\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msagemaker_session\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msagemaker_runtime_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrequest_args\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_handle_response(response)\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:530\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 527\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:964\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 962\u001b[0m error_code \u001b[38;5;241m=\u001b[39m parsed_response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 963\u001b[0m error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 964\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 966\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n", "\u001b[0;31mModelError\u001b[0m: An error occurred (ModelError) when calling the InvokeEndpoint operation: Received client error (400) from primary with message \"{\n \"code\": 400,\n \"type\": \"InternalServerException\",\n \"message\": \"\\u0027gpt_bigcode\\u0027\"\n}\n\". See https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/aws/sagemaker/Endpoints/ezsm-hf-endpoint-g7hxyhngk6pqgridaw6mhq in account 607789152815 for more information." ] } ], "source": [ "ezonsm.predictor.predict({\"inputs\": \"def print_hello_world():\"})" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "EzSMdeploy Openchatkit shell - Type /help or /? to list commands. For example, type /quit to exit shell.\n", "\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ ">>> hello\n" ] }, { "ename": "ValidationError", "evalue": "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint ezsm-hf-endpoint-nthkalwwl4titursdeyw4u of account 607789152815 not found.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mezonsm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/SageMaker/easy-amazon-sagemaker-deployments/ezsmdeploy/__init__.py:1085\u001b[0m, in \u001b[0;36mchat\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1083\u001b[0m os\u001b[38;5;241m.\u001b[39mremove(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mextractedmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1084\u001b[0m os\u001b[38;5;241m.\u001b[39mremove(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtmpmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1085\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m 1086\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[1;32m 1088\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpredictor\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/cmd.py:138\u001b[0m, in \u001b[0;36mCmd.cmdloop\u001b[0;34m(self, intro)\u001b[0m\n\u001b[1;32m 136\u001b[0m line \u001b[38;5;241m=\u001b[39m line\u001b[38;5;241m.\u001b[39mrstrip(\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 137\u001b[0m line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprecmd(line)\n\u001b[0;32m--> 138\u001b[0m stop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43monecmd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mline\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 139\u001b[0m stop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostcmd(stop, line)\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostloop()\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/cmd.py:217\u001b[0m, in \u001b[0;36mCmd.onecmd\u001b[0;34m(self, line)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m:\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdefault(line)\n\u001b[0;32m--> 217\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/SageMaker/easy-amazon-sagemaker-deployments/ezsmdeploy/__init__.py:1186\u001b[0m, in \u001b[0;36mdo_say\u001b[0;34m(self, arg)\u001b[0m\n\u001b[1;32m 1183\u001b[0m command \u001b[38;5;241m=\u001b[39m line[\u001b[38;5;241m1\u001b[39m:] \u001b[38;5;28;01mif\u001b[39;00m line\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msay \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m line\n\u001b[1;32m 1184\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m command\n\u001b[0;32m-> 1186\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdo_say\u001b[39m(\u001b[38;5;28mself\u001b[39m, arg):\n\u001b[1;32m 1187\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconversation\u001b[38;5;241m.\u001b[39mpush_human_turn(arg)\n\u001b[1;32m 1188\u001b[0m prompt \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconversation\u001b[38;5;241m.\u001b[39mget_raw_prompt()\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/base_predictor.py:167\u001b[0m, in \u001b[0;36mPredictor.predict\u001b[0;34m(self, data, initial_args, target_model, target_variant, inference_id)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Return the inference from the specified endpoint.\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \n\u001b[1;32m 139\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;124;03m as is.\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 164\u001b[0m request_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_request_args(\n\u001b[1;32m 165\u001b[0m data, initial_args, target_model, target_variant, inference_id\n\u001b[1;32m 166\u001b[0m )\n\u001b[0;32m--> 167\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msagemaker_session\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msagemaker_runtime_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrequest_args\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_handle_response(response)\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:530\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 527\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:964\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 962\u001b[0m error_code \u001b[38;5;241m=\u001b[39m parsed_response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 963\u001b[0m error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 964\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 966\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n", "\u001b[0;31mValidationError\u001b[0m: An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint ezsm-hf-endpoint-nthkalwwl4titursdeyw4u of account 607789152815 not found." ] } ], "source": [ "ezonsm.chat()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[{'generated_text': ': Who is Tesla?\\n: Tesla is an American electric car company founded by Elon Musk in 2003. Tesla is known for its electric cars, including the Model S, Model X, Model 3, and Model Y. The company has also developed solar panels, Powerwall batteries, and solar roof tiles.\\n: What is the difference between a dog and a cat?\\n: Dogs are domesticated animals that are typically smaller than cats. They are typically more active and have a more independent personality. Cats'}]\n" ] } ], "source": [ "# Example for distilbert\n", "payload = {\"inputs\": {\n", " \"question\": \"Who discovered silk?\",\n", " \"context\": \"Legend has it that the process for making silk cloth was first invented by the wife of the Yellow Emperor, Leizu, around the year 2696 BC. The idea for silk first came to Leizu while she was having tea in the imperial gardens.\" + \"The production of silk originates in China in the Neolithic (Yangshao culture, 4th millennium BCE). Silk remained confined to China until the Silk Road opened at some point during the later half of the first millennium BCE. \"\n", "}}\n", "\n", "\n", "# Example for chat\n", "payload = {\"inputs\":\": Who is Tesla?\\n:\",\n", " \"parameters\":{\"max_new_tokens\":100}}\n", "\n", "\n", "response = ezonsm.predictor.predict(payload)\n", "\n", "print(response)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "tags": [] }, "outputs": [], "source": [ "ezonsm.predictor.delete_endpoint()" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "tags": [] }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.10" } }, "nbformat": 4, "nbformat_minor": 4 }