{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "750440d5", "metadata": {}, "outputs": [], "source": [ "%reload_ext autoreload\n", "%autoreload 2\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "id": "1296f28e", "metadata": {}, "source": [ "# TorchServe" ] }, { "cell_type": "code", "execution_count": null, "id": "63d7ff5a", "metadata": {}, "outputs": [], "source": [ "!pygmentize ../deployment/handler.py" ] }, { "cell_type": "code", "execution_count": null, "id": "b0ebe88e", "metadata": {}, "outputs": [], "source": [ "%%bash\n", "cd ../\n", "\n", "cp model/resnet50_0.962_head.pth model/head_weight.pth\n", "cp model/resnet50_0.962_encoder.pth model/encoder_weight.pth\n", "\n", "torch-model-archiver --model-name twin \\\n", "--version 1.0 --serialized-file ./model/encoder_weight.pth \\\n", "--export-path model_store --handler ./deployment/handler.py \\\n", "-f --extra-files ./model/head_weight.pth\n", "\n", "ls -lh ./model_store/" ] }, { "cell_type": "markdown", "id": "e67a8629", "metadata": {}, "source": [ "# Amazon SageMaker" ] }, { "cell_type": "markdown", "id": "1869d402", "metadata": {}, "source": [ "## Boilerplate" ] }, { "cell_type": "code", "execution_count": null, "id": "1cbd1180", "metadata": {}, "outputs": [], "source": [ "# !pip install boto3\n", "# !pip install sagemaker" ] }, { "cell_type": "code", "execution_count": null, "id": "272e2572", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import boto3, time" ] }, { "cell_type": "markdown", "id": "4d4f69ac", "metadata": {}, "source": [ "### Session" ] }, { "cell_type": "code", "execution_count": null, "id": "68e9ffb9", "metadata": {}, "outputs": [], "source": [ "sess = boto3.Session()\n", "sm = sess.client(\"sagemaker\")\n", "region = sess.region_name\n", "\n", "account = boto3.client(\"sts\").get_caller_identity().get(\"Account\")\n", "region, account" ] }, { "cell_type": "markdown", "id": "0b627565", "metadata": {}, "source": [ "### IAM Role\n", "\n", "Note: make sure the IAM role has:\n", "- `AmazonS3FullAccess`\n", "- `AmazonEC2ContainerRegistryFullAccess`\n", "- `AmazonSageMakerFullAccess`" ] }, { "cell_type": "code", "execution_count": null, "id": "282fcf08", "metadata": {}, "outputs": [], "source": [ "import sagemaker\n", "\n", "role = sagemaker.get_execution_role()\n", "role" ] }, { "cell_type": "markdown", "id": "b79fafc1", "metadata": {}, "source": [ "## Amazon Elastic Container Registry (ECR)\n", "\n", "**Note**: create ECR if it doesn’t exist" ] }, { "cell_type": "code", "execution_count": null, "id": "ee78d2e9", "metadata": {}, "outputs": [], "source": [ "registry_name = \"twin-pytorch\"\n", "image = f\"{account}.dkr.ecr.{region}.amazonaws.com/{registry_name}:latest\"\n", "image" ] }, { "cell_type": "code", "execution_count": null, "id": "5b96b490", "metadata": {}, "outputs": [], "source": [ "!aws ecr create-repository --repository-name {registry_name} --region {region}" ] }, { "cell_type": "markdown", "id": "b1d233cc", "metadata": {}, "source": [ "### Pytorch Model Artifact\n", "\n", "Create a compressed `*.tar.gz` file from the `*.mar` file per requirement of Amazon SageMaker and upload the model to your Amazon S3 bucket." ] }, { "cell_type": "code", "execution_count": null, "id": "fa0b2654", "metadata": {}, "outputs": [], "source": [ "model_file_name = \"twin\"\n", "s3_bucket_name = \"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "d5017491", "metadata": {}, "outputs": [], "source": [ "%%bash -s \"$model_file_name\" \"$s3_bucket_name\"\n", "cd ../model_store/\n", "tar cvfz $1.tar.gz $1.mar\n", "aws s3 cp $1.tar.gz s3://$2/" ] }, { "cell_type": "markdown", "id": "e6da15ab", "metadata": {}, "source": [ "### Build TorchServe Docker container and push it to Amazon ECR" ] }, { "cell_type": "code", "execution_count": null, "id": "450d7bb1", "metadata": {}, "outputs": [], "source": [ "!aws ecr get-login-password --region {region} | docker login --username AWS --password-stdin {account}.dkr.ecr.{region}.amazonaws.com" ] }, { "cell_type": "code", "execution_count": null, "id": "dcc8d810", "metadata": {}, "outputs": [], "source": [ "%%bash -s \"$registry_name\" \"$image\"\n", "cd ../\n", "docker build -t $1 .\n", "docker tag $1 $2\n", "docker push $2" ] }, { "cell_type": "markdown", "id": "e98765f1", "metadata": {}, "source": [ "### SageMaker Model" ] }, { "cell_type": "code", "execution_count": null, "id": "a7cb6de6", "metadata": {}, "outputs": [], "source": [ "model_data = f\"s3://{s3_bucket_name}/{model_file_name}.tar.gz\"\n", "sm_model_name = \"torchserve-twin-v1\"\n", "\n", "container = {\"Image\": image, \"ModelDataUrl\": model_data}\n", "\n", "create_model_response = sm.create_model(\n", " ModelName=sm_model_name, ExecutionRoleArn=role, PrimaryContainer=container\n", ")\n", "\n", "print(create_model_response[\"ModelArn\"])" ] }, { "cell_type": "markdown", "id": "95793502", "metadata": {}, "source": [ "## Inference Endpoint\n", "\n", "Configuration with **InstanceType** and **Model Monitoring**" ] }, { "cell_type": "code", "execution_count": null, "id": "ed85e1be", "metadata": {}, "outputs": [], "source": [ "endpoint_config_name = \"torchserve-endpoint-config-\" + time.strftime(\n", " \"%Y-%m-%d-%H-%M-%S\", time.gmtime()\n", ")\n", "print(endpoint_config_name)\n", "\n", "create_endpoint_config_response = sm.create_endpoint_config(\n", " EndpointConfigName=endpoint_config_name,\n", " ProductionVariants=[\n", " {\n", " \"InstanceType\": \"ml.g4dn.xlarge\", # Choose Your Preferred Instance Type\n", " \"InitialVariantWeight\": 1,\n", " \"InitialInstanceCount\": 1,\n", " \"ModelName\": sm_model_name,\n", " \"VariantName\": \"AllTraffic\",\n", " }\n", " ],\n", " # DataCaptureConfig={\n", " # \"EnableCapture\": True,\n", " # \"InitialSamplingPercentage\": 100,\n", " # \"DestinationS3Uri\": f\"s3://{s3_bucket_name}/monitor/\",\n", " # \"CaptureOptions\": [\n", " # {\"CaptureMode\": \"Input\"},\n", " # {\"CaptureMode\": \"Output\"},\n", " # ],\n", " # },\n", ")\n", "\n", "print(\"Endpoint Config Arn: \" + create_endpoint_config_response[\"EndpointConfigArn\"])" ] }, { "cell_type": "markdown", "id": "ce8f6f04", "metadata": {}, "source": [ "### Endpoint" ] }, { "cell_type": "code", "execution_count": null, "id": "31017029", "metadata": {}, "outputs": [], "source": [ "endpoint_name = \"torchserve-endpoint-\" + time.strftime(\n", " \"%Y-%m-%d-%H-%M-%S\", time.gmtime()\n", ")\n", "print(endpoint_name)\n", "\n", "create_endpoint_response = sm.create_endpoint(\n", " EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name\n", ")\n", "print(create_endpoint_response[\"EndpointArn\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "f1e1d96f", "metadata": {}, "outputs": [], "source": [ "%%time\n", "resp = sm.describe_endpoint(EndpointName=endpoint_name)\n", "status = resp[\"EndpointStatus\"]\n", "print(\"Status: \" + status)\n", "\n", "while status == \"Creating\":\n", " time.sleep(60)\n", " resp = sm.describe_endpoint(EndpointName=endpoint_name)\n", " status = resp[\"EndpointStatus\"]\n", " print(\"Status: \" + status)\n", "\n", "print(\"Arn: \" + resp[\"EndpointArn\"])\n", "print(\"Status: \" + status)" ] }, { "cell_type": "markdown", "id": "4aca1975", "metadata": {}, "source": [ "### Testing" ] }, { "cell_type": "code", "execution_count": null, "id": "966c3871", "metadata": {}, "outputs": [], "source": [ "cam = True\n", "r = requests.Request(\n", " \"POST\",\n", " \"http://localhost:8080/invocations\",\n", " files={\n", " \"left\": open(\"../sample/c1.jpg\", \"rb\"),\n", " \"right\": open(\"../sample/c3.jpg\", \"rb\"),\n", " },\n", " data={\"cam\": str(cam)}\n", ")\n", "r = r.prepare()\n", "content_type = r.headers[\"Content-Type\"]\n", "payload = r.body\n", "content_type, type(payload)" ] }, { "cell_type": "code", "execution_count": null, "id": "ab69fdd4", "metadata": {}, "outputs": [], "source": [ "client = boto3.client(\"sagemaker-runtime\")\n", "response = client.invoke_endpoint(\n", " EndpointName=endpoint_name, ContentType=content_type, Body=payload\n", " )\n", "res = response[\"Body\"].read()\n", "neg, pos, *maps = eval(res)\n", "neg, pos" ] }, { "cell_type": "code", "execution_count": null, "id": "b0216c46", "metadata": {}, "outputs": [], "source": [ "import torch\n", "import matplotlib.pyplot as plt\n", "\n", "\n", "if cam:\n", " length = len(maps)\n", " cam_map_left, cam_map_right = maps[: length // 2], maps[length // 2 :]\n", "\n", " cam_map_left = torch.tensor(cam_map_left)\n", " cam_map_right = torch.tensor(cam_map_right)\n", "\n", " _, ax = plt.subplots(1, 2, figsize=(10, 5))\n", " ax[0].imshow(\n", " cam_map_left,\n", " alpha=0.6,\n", " extent=(0, 224, 224, 0),\n", " interpolation=\"bilinear\",\n", " cmap=\"jet\",\n", " )\n", " ax[0].axis(\"off\")\n", "\n", " ax[1].imshow(\n", " cam_map_right,\n", " alpha=0.6,\n", " extent=(0, 224, 224, 0),\n", " interpolation=\"bilinear\",\n", " cmap=\"jet\",\n", " )\n", " ax[1].axis(\"off\")\n", "\n", " plt.show()" ] }, { "cell_type": "markdown", "id": "8b96b278", "metadata": {}, "source": [ "## Cleanup" ] }, { "cell_type": "code", "execution_count": null, "id": "a4dff7cf", "metadata": {}, "outputs": [], "source": [ "# client = boto3.client(\"sagemaker\")\n", "# client.delete_endpoint(EndpointName=endpoint_name)\n", "# client.delete_endpoint_config(EndpointConfigName=endpoint_config_name)\n", "# client.delete_model(ModelName=sm_model_name)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" }, "latex_envs": { "LaTeX_envs_menu_present": true, "autoclose": false, "autocomplete": true, "bibliofile": "biblio.bib", "cite_by": "apalike", "current_citInitial": 1, "eqLabelWithNumbers": true, "eqNumInitial": 1, "hotkeys": { "equation": "Ctrl-E", "itemize": "Ctrl-I" }, "labels_anchors": false, "latex_user_defs": false, "report_style_numbering": false, "user_envs_cfg": false }, "varInspector": { "cols": { "lenName": 16, "lenType": 16, "lenVar": 40 }, "kernels_config": { "python": { "delete_cmd_postfix": "", "delete_cmd_prefix": "del ", "library": "var_list.py", "varRefreshCmd": "print(var_dic_list())" }, "r": { "delete_cmd_postfix": ") ", "delete_cmd_prefix": "rm(", "library": "var_list.r", "varRefreshCmd": "cat(var_dic_list()) " } }, "types_to_exclude": [ "module", "function", "builtin_function_or_method", "instance", "_Feature" ], "window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }