{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%reload_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Amazon SageMaker" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import base64\n", "import json\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from PIL import Image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Boilerplate" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Session" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import boto3, time, json\n", "\n", "sess = boto3.Session()\n", "sm = sess.client(\"sagemaker\")\n", "region = sess.region_name\n", "account = boto3.client(\"sts\").get_caller_identity().get(\"Account\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### IAM Role\n", "\n", "**Note**: make sure the IAM role has: \n", "- `AmazonS3FullAccess` \n", "- `AmazonEC2ContainerRegistryFullAccess` \n", "- `AmazonSageMakerFullAccess` " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sagemaker\n", "\n", "role = sagemaker.get_execution_role()\n", "role" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Amazon Elastic Container Registry (ECR)\n", "\n", "**Note**: create ECR if it doesn't exist" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "registry_name = \"fastai-torchserve-sagemaker\"\n", "# !aws ecr create-repository --repository-name {registry_name}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "image = f\"{account}.dkr.ecr.{region}.amazonaws.com/{registry_name}:latest\"\n", "image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Pytorch Model Artifact\n", "\n", "Create a compressed `*.tar.gz` file from the `*.mar` file per requirement of Amazon SageMaker and upload the model to your Amazon S3 bucket." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model_file_name = \"fastunet\"\n", "s3_bucket_name = \"\"\n", "# !tar cvzf {model_file_name}.tar.gz fastunet.mar\n", "# !aws s3 cp {model_file_name}.tar.gz s3://{s3_bucket_name}/" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Build a FastAI+TorchServe Docker container and push it to Amazon ECR" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!aws ecr get-login-password --region {region} | docker login --username AWS --password-stdin {account}.dkr.ecr.{region}.amazonaws.com\n", "!docker build -t {registry_name} ../\n", "!docker tag {registry_name}:latest {image}\n", "!docker push {image}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model_data = f\"s3://{s3_bucket_name}/{model_file_name}.tar.gz\"\n", "sm_model_name = \"fastai-unet-torchserve-sagemaker\"\n", "\n", "container = {\"Image\": image, \"ModelDataUrl\": model_data}\n", "\n", "create_model_response = sm.create_model(\n", " ModelName=sm_model_name, ExecutionRoleArn=role, PrimaryContainer=container\n", ")\n", "\n", "print(create_model_response[\"ModelArn\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Batch Transform" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### S3 Input and Output" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "batch_input = f\"s3://{s3_bucket_name}/batch_transform_fastai_torchserve_sagemaker/\"\n", "batch_output = f\"s3://{s3_bucket_name}/batch_transform_fastai_torchserve_sagemaker_output/\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!aws s3 ls {batch_input}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import time\n", "\n", "batch_job_name = 'fastunet-batch' + time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.gmtime())\n", "batch_job_name" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Batch transform jobs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "request = {\n", " \"ModelClientConfig\": {\n", " \"InvocationsTimeoutInSeconds\": 3600,\n", " \"InvocationsMaxRetries\": 1,\n", " },\n", " \"TransformJobName\": batch_job_name,\n", " \"ModelName\": sm_model_name,\n", " \"BatchStrategy\": \"MultiRecord\",\n", " \"TransformOutput\": {\"S3OutputPath\": batch_output, \"AssembleWith\": \"Line\"},\n", " \"TransformInput\": {\n", " \"DataSource\": {\n", " \"S3DataSource\": {\"S3DataType\": \"S3Prefix\", \"S3Uri\": batch_input}\n", " },\n", " \"CompressionType\": \"None\",\n", " },\n", " \"TransformResources\": {\"InstanceType\": \"ml.p2.xlarge\", \"InstanceCount\": 1},\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "sm.create_transform_job(**request)\n", "\n", "while True:\n", " response = sm.describe_transform_job(TransformJobName=batch_job_name)\n", " status = response[\"TransformJobStatus\"]\n", " if status == \"Completed\":\n", " print(\"Transform job ended with status: \" + status)\n", " break\n", " if status == \"Failed\":\n", " message = response[\"FailureReason\"]\n", " print(\"Transform failed with the following error: {}\".format(message))\n", " raise Exception(\"Transform job failed\")\n", " print(\"Transform job is still in status: \" + status)\n", " time.sleep(30)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Testing" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "s3 = boto3.resource(\"s3\")\n", "s3.Bucket(f\"{s3_bucket_name}\").download_file(\n", " \"batch_transform_fastai_torchserve_sagemaker_output/street_view_of_a_small_neighborhood.png.out\",\n", " \"street_view_of_a_small_neighborhood.txt\",\n", ")\n", "s3.Bucket(f\"{s3_bucket_name}\").download_file(\n", " \"batch_transform_fastai_torchserve_sagemaker/street_view_of_a_small_neighborhood.png\",\n", " \"street_view_of_a_small_neighborhood.png\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "with open(\"street_view_of_a_small_neighborhood.txt\") as f:\n", " results = f.read()\n", "\n", "response = json.loads(results)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pred_decoded_byte = base64.decodebytes(bytes(response[\"base64_prediction\"], encoding=\"utf-8\"))\n", "pred_decoded = np.reshape(\n", " np.frombuffer(pred_decoded_byte, dtype=np.uint8), (96, 128)\n", ")\n", "plt.imshow(pred_decoded);" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Inference Endpoint" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Endpoint configuration\n", "\n", "**Note**: choose your preferred `InstanceType`: https://aws.amazon.com/sagemaker/pricing/" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import time\n", "\n", "endpoint_config_name = \"torchserve-endpoint-config-\" + time.strftime(\n", " \"%Y-%m-%d-%H-%M-%S\", time.gmtime()\n", ")\n", "print(endpoint_config_name)\n", "\n", "create_endpoint_config_response = sm.create_endpoint_config(\n", " EndpointConfigName=endpoint_config_name,\n", " ProductionVariants=[\n", " {\n", " \"InstanceType\": \"ml.g4dn.xlarge\",\n", " \"InitialVariantWeight\": 1,\n", " \"InitialInstanceCount\": 1,\n", " \"ModelName\": sm_model_name,\n", " \"VariantName\": \"AllTraffic\",\n", " }\n", " ],\n", ")\n", "\n", "print(\"Endpoint Config Arn: \" + create_endpoint_config_response[\"EndpointConfigArn\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Endpoint" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "endpoint_name = \"fastunet-torchserve-endpoint-\" + time.strftime(\n", " \"%Y-%m-%d-%H-%M-%S\", time.gmtime()\n", ")\n", "print(endpoint_name)\n", "\n", "create_endpoint_response = sm.create_endpoint(\n", " EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name\n", ")\n", "print(create_endpoint_response[\"EndpointArn\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "resp = sm.describe_endpoint(EndpointName=endpoint_name)\n", "status = resp[\"EndpointStatus\"]\n", "print(\"Status: \" + status)\n", "\n", "while status == \"Creating\":\n", " time.sleep(60)\n", " resp = sm.describe_endpoint(EndpointName=endpoint_name)\n", " status = resp[\"EndpointStatus\"]\n", " print(\"Status: \" + status)\n", "\n", "print(\"Arn: \" + resp[\"EndpointArn\"])\n", "print(\"Status: \" + status)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Testing" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "file_name = \"../sample/street_view_of_a_small_neighborhood.png\"\n", "\n", "with open(file_name, 'rb') as f:\n", " payload = f.read()\n", " \n", "Image.open(file_name)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "client = boto3.client(\"runtime.sagemaker\")\n", "response = client.invoke_endpoint(\n", " EndpointName=endpoint_name, ContentType=\"application/x-image\", Body=payload\n", ")\n", "response = json.loads(response[\"Body\"].read())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pred_decoded_byte = base64.decodebytes(bytes(response[\"base64_prediction\"], encoding=\"utf-8\"))\n", "pred_decoded = np.reshape(\n", " np.frombuffer(pred_decoded_byte, dtype=np.uint8), (96, 128)\n", ")\n", "plt.imshow(pred_decoded);" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Cleanup" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "client = boto3.client(\"sagemaker\")\n", "client.delete_model(ModelName=sm_model_name)\n", "client.delete_endpoint(EndpointName=endpoint_name)\n", "client.delete_endpoint_config(EndpointConfigName=endpoint_config_name)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false }, "varInspector": { "cols": { "lenName": 16, "lenType": 16, "lenVar": 40 }, "kernels_config": { "python": { "delete_cmd_postfix": "", "delete_cmd_prefix": "del ", "library": "var_list.py", "varRefreshCmd": "print(var_dic_list())" }, "r": { "delete_cmd_postfix": ") ", "delete_cmd_prefix": "rm(", "library": "var_list.r", "varRefreshCmd": "cat(var_dic_list()) " } }, "types_to_exclude": [ "module", "function", "builtin_function_or_method", "instance", "_Feature" ], "window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }