{ "cells": [ { "cell_type": "markdown", "id": "ab3fb8b5-37a4-4aae-8e1a-0ebc99ef1eac", "metadata": { "tags": [] }, "source": [ "# 1. Rekognition Re-Index Solution" ] }, { "cell_type": "markdown", "id": "26bea671-a45f-4d54-a685-85b94e91f3cc", "metadata": {}, "source": [ "This notebook contains scripts to help you deploy and use the solution.\n", "\n", "⚠️ **Please use notebook 0-Data-Preparation before running this notebook** ⚠️ \n", "\n", "**Notebook Steps:**\n", "1. Import libraries and clients\n", "2. Define variables\n", "3. Deploy and launch the solution\n", "4. Analyze the results" ] }, { "cell_type": "markdown", "id": "c374bb0a-b5bf-454b-881c-c36a6ddea3f4", "metadata": { "tags": [] }, "source": [ "## 1. Import Libraries and Clients" ] }, { "cell_type": "code", "execution_count": null, "id": "705a2c73-b630-4d40-8969-04f0bcb63084", "metadata": { "tags": [] }, "outputs": [], "source": [ "import boto3, json, os, logging, sagemaker, time\n", "from botocore.exceptions import ClientError\n", "from zipfile import ZipFile\n", "rekClient = boto3.client('rekognition')\n", "dynamoClient = boto3.client('dynamodb')\n", "sfClient = boto3.client('stepfunctions')\n", "cfClient = boto3.client('cloudformation')\n", "sm_session = sagemaker.Session()\n", "boto3_session = boto3.session.Session()\n", "boto3_region = boto3_session.region_name" ] }, { "cell_type": "markdown", "id": "6aac8527-b15e-41a0-a31b-0ad6c521d231", "metadata": { "tags": [] }, "source": [ "## 2. Define Variables" ] }, { "cell_type": "markdown", "id": "f44d8bb5-88a7-4ddb-bf00-f5a52f4e3cb8", "metadata": {}, "source": [ "If you have run the previous notebook you will find the details in the last cell." ] }, { "cell_type": "code", "execution_count": null, "id": "b0a03f9a-78a3-4e77-a812-97c2baef1ee2", "metadata": { "tags": [] }, "outputs": [], "source": [ "old_CollectionId =\"\"\n", "new_CollectionId =\"\"\n", "s3_bucket_name =\"\"\n", "records_folder =\"\"\n", "records_filename =\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "913b86ce-a1fa-42e6-9ca7-5f4e804a5405", "metadata": { "tags": [] }, "outputs": [], "source": [ "stack_name = \"\" ## Specify a unique name for the stack\n", "RekognitionIndexFacesQualityFilter = \"AUTO\"\n", "RekognitionIndexFacesTPSLimit = \"50\"" ] }, { "cell_type": "markdown", "id": "2f4362ff-3271-46d5-9519-49e15031458b", "metadata": { "tags": [] }, "source": [ "## 3. Deploy and launch the solution" ] }, { "cell_type": "markdown", "id": "ac0ac5d0-092d-4377-9106-4cba2a86d686", "metadata": { "tags": [] }, "source": [ "#### 3.1 Prepare and upload the necessary assets " ] }, { "cell_type": "code", "execution_count": null, "id": "b7a74f6e-4478-4d76-9890-d93117fb1455", "metadata": {}, "outputs": [], "source": [ "lambda_folder = \"assets/lambda_functions\"\n", "for file in os.listdir(lambda_folder):\n", " root,ext = os.path.splitext(file)\n", " if ext == \".py\":\n", " with ZipFile(\"{}/{}.zip\".format(lambda_folder,root),'w') as zip:\n", " zip.write(\"{}/{}\".format(lambda_folder,file),file)\n", " zip.close()\n", " asset_s3_uri = sm_session.upload_data(\"{}/{}.zip\".format(lambda_folder,root), s3_bucket_name, lambda_folder)\n", " print(\"Your function asset is located in {}\".format(asset_s3_uri))" ] }, { "cell_type": "markdown", "id": "9662efcf-02e1-41b8-938f-fca639bd03d3", "metadata": { "tags": [] }, "source": [ "#### 3.2 Deploy the Amazon CloudFormation template " ] }, { "cell_type": "markdown", "id": "99c0c825-5f6a-4f18-8e64-3f56fa0cd665", "metadata": { "tags": [] }, "source": [ "The cloudformation stack deployment should take around 3 minutes." ] }, { "cell_type": "code", "execution_count": null, "id": "28264b26-db2a-4929-95d4-8eadf301a545", "metadata": { "tags": [] }, "outputs": [], "source": [ "with open('{}/{}'.format(\"assets/cloudformation_template\",\"template.yaml\"), 'r') as file:\n", " template_body = file.read()\n", " file.close()\n", " \n", "response_cft = cfClient.create_stack(\n", " StackName= stack_name,\n", " TemplateBody= template_body,\n", " Parameters=[\n", " {\n", " 'ParameterKey': 'AssetsBucket',\n", " 'ParameterValue': s3_bucket_name,\n", " },\n", " {\n", " 'ParameterKey': 'RekognitionIndexFacesQualityFilter',\n", " 'ParameterValue': RekognitionIndexFacesQualityFilter,\n", " },\n", " {\n", " 'ParameterKey': 'RekognitionIndexFacesTPSLimit',\n", " 'ParameterValue': RekognitionIndexFacesTPSLimit,\n", " },\n", " ],\n", " Capabilities=['CAPABILITY_IAM','CAPABILITY_NAMED_IAM','CAPABILITY_AUTO_EXPAND'],\n", " TimeoutInMinutes=20,\n", " OnFailure='ROLLBACK'\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "3f297add-9cb5-4943-aabc-d0a885eb4690", "metadata": { "tags": [] }, "outputs": [], "source": [ "def get_status(stackId):\n", " cft_status = cfClient.describe_stacks(\n", " StackName=stackId\n", " )\n", " return cft_status[\"Stacks\"][0]" ] }, { "cell_type": "code", "execution_count": null, "id": "a8725af5-7d2d-472f-aa9f-232c7e23d62b", "metadata": { "tags": [] }, "outputs": [], "source": [ "print(\"Creating Stack...\")\n", "while get_status(response_cft[\"StackId\"])[\"StackStatus\"] == \"CREATE_IN_PROGRESS\":\n", " print(\".\", end =\"\")\n", " time.sleep(10)\n", "print(\"Stack finished launching with state:\", get_status(response_cft[\"StackId\"])[\"StackStatus\"])\n", "sf_arn = get_status(response_cft[\"StackId\"])[\"Outputs\"][0][\"OutputValue\"]\n", "print(\"Step Function Workflow ARN:\", sf_arn)" ] }, { "cell_type": "markdown", "id": "a70fab62-9260-402e-a774-1b5efe322208", "metadata": { "tags": [] }, "source": [ "#### 3.3 Launch a new Step Functions Workflow to ReIndex the collection" ] }, { "cell_type": "code", "execution_count": null, "id": "ee906b53-76e7-4118-92a8-e97c5df5bee2", "metadata": {}, "outputs": [], "source": [ "execution_response = sfClient.start_execution(\n", " stateMachineArn=sf_arn,\n", " input=json.dumps({\"bucket\": s3_bucket_name,\"key\": \"{}/{}\".format(records_folder,records_filename)})\n", ")\n", "execution_arn = execution_response[\"executionArn\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "52d9cdca-397f-4c8f-9288-0fbd75f0832c", "metadata": { "tags": [] }, "outputs": [], "source": [ "def get_execution_status(execution_arn):\n", " execution_status = sfClient.describe_execution(\n", " executionArn=execution_arn\n", " )\n", " return execution_status[\"status\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "8350d7e5-386d-4eaf-bc83-0d595102e9e9", "metadata": { "tags": [] }, "outputs": [], "source": [ "print(\"Reindexing face collection...\")\n", "while get_execution_status(execution_arn) not in [\"SUCCEEDED\",\"FAILED\"]:\n", " print(\".\", end =\"\")\n", " time.sleep(10)\n", "print(\"Execution finished launching with state:\", get_execution_status(execution_arn))" ] }, { "cell_type": "markdown", "id": "17819c76-765f-4cd2-ad33-514f5f190a8a", "metadata": { "tags": [] }, "source": [ "## 4. Analyze the results" ] }, { "cell_type": "code", "execution_count": null, "id": "02eef5da-9818-446b-a9a4-f1d6386ebb63", "metadata": { "tags": [] }, "outputs": [], "source": [ "def list_collection_faces(collection_id):\n", " response = rekClient.list_faces(\n", " CollectionId=collection_id\n", " )\n", " print(\"Number Faces in {} is : {} \".format(collection_id, len(response[\"Faces\"]))) " ] }, { "cell_type": "code", "execution_count": null, "id": "aa4829f2-7883-46b0-9e72-c531cae987da", "metadata": { "tags": [] }, "outputs": [], "source": [ "list_collection_faces(old_CollectionId)\n", "list_collection_faces(new_CollectionId)" ] }, { "cell_type": "markdown", "id": "3c22c0ff-978c-4b3a-b945-fcea210d2169", "metadata": { "tags": [] }, "source": [ "#### Analyze Results" ] }, { "cell_type": "code", "execution_count": null, "id": "491acb68-78b0-42a8-84da-8c49c7513ada", "metadata": { "tags": [] }, "outputs": [], "source": [ "response_scan = dynamoClient.scan(\n", " TableName=get_status(response_cft[\"StackId\"])[\"Outputs\"][1][\"OutputValue\"]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "1db49639-5b3d-4539-9c6e-bc6674cb5093", "metadata": { "tags": [] }, "outputs": [], "source": [ "new_faces_found = 0\n", "for face in response_scan[\"Items\"]:\n", " if (face[\"IsNewFace\"][\"S\"]) == True:\n", " new_faces_found = new_faces_found +1" ] }, { "cell_type": "code", "execution_count": null, "id": "c0aa263f-9532-4547-8b60-8b5da859bb3d", "metadata": { "tags": [] }, "outputs": [], "source": [ "print(\"Total faces reindexed:\",response_scan[\"Count\"])\n", "print(\"New faces found:\",new_faces_found)" ] } ], "metadata": { "availableInstances": [ { "_defaultOrder": 0, "_isFastLaunch": true, "category": "General purpose", "gpuNum": 0, "memoryGiB": 4, "name": "ml.t3.medium", "vcpuNum": 2 }, { "_defaultOrder": 1, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 8, "name": "ml.t3.large", "vcpuNum": 2 }, { "_defaultOrder": 2, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 16, "name": "ml.t3.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 3, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 32, "name": "ml.t3.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 4, "_isFastLaunch": true, "category": "General purpose", "gpuNum": 0, "memoryGiB": 8, "name": "ml.m5.large", "vcpuNum": 2 }, { "_defaultOrder": 5, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 16, "name": "ml.m5.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 6, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 32, "name": "ml.m5.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 7, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 64, "name": "ml.m5.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 8, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 128, "name": "ml.m5.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 9, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 192, "name": "ml.m5.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 10, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 256, "name": "ml.m5.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 11, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 384, "name": "ml.m5.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 12, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 8, "name": "ml.m5d.large", "vcpuNum": 2 }, { "_defaultOrder": 13, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 16, "name": "ml.m5d.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 14, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 32, "name": "ml.m5d.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 15, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 64, "name": "ml.m5d.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 16, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 128, "name": "ml.m5d.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 17, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 192, "name": "ml.m5d.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 18, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 256, "name": "ml.m5d.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 19, "_isFastLaunch": false, "category": "General purpose", "gpuNum": 0, "memoryGiB": 384, "name": "ml.m5d.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 20, "_isFastLaunch": true, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 4, "name": "ml.c5.large", "vcpuNum": 2 }, { "_defaultOrder": 21, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 8, "name": "ml.c5.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 22, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 16, "name": "ml.c5.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 23, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 32, "name": "ml.c5.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 24, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 72, "name": "ml.c5.9xlarge", "vcpuNum": 36 }, { "_defaultOrder": 25, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 96, "name": "ml.c5.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 26, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 144, "name": "ml.c5.18xlarge", "vcpuNum": 72 }, { "_defaultOrder": 27, "_isFastLaunch": false, "category": "Compute optimized", "gpuNum": 0, "memoryGiB": 192, "name": "ml.c5.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 28, "_isFastLaunch": true, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 16, "name": "ml.g4dn.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 29, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 32, "name": "ml.g4dn.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 30, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 64, "name": "ml.g4dn.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 31, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 128, "name": "ml.g4dn.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 32, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 4, "memoryGiB": 192, "name": "ml.g4dn.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 33, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 256, "name": "ml.g4dn.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 34, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 61, "name": "ml.p3.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 35, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 4, "memoryGiB": 244, "name": "ml.p3.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 36, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 8, "memoryGiB": 488, "name": "ml.p3.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 37, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 8, "memoryGiB": 768, "name": "ml.p3dn.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 38, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 16, "name": "ml.r5.large", "vcpuNum": 2 }, { "_defaultOrder": 39, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 32, "name": "ml.r5.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 40, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 64, "name": "ml.r5.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 41, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 128, "name": "ml.r5.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 42, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 256, "name": "ml.r5.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 43, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 384, "name": "ml.r5.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 44, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 512, "name": "ml.r5.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 45, "_isFastLaunch": false, "category": "Memory Optimized", "gpuNum": 0, "memoryGiB": 768, "name": "ml.r5.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 46, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 16, "name": "ml.g5.xlarge", "vcpuNum": 4 }, { "_defaultOrder": 47, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 32, "name": "ml.g5.2xlarge", "vcpuNum": 8 }, { "_defaultOrder": 48, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 64, "name": "ml.g5.4xlarge", "vcpuNum": 16 }, { "_defaultOrder": 49, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 128, "name": "ml.g5.8xlarge", "vcpuNum": 32 }, { "_defaultOrder": 50, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 1, "memoryGiB": 256, "name": "ml.g5.16xlarge", "vcpuNum": 64 }, { "_defaultOrder": 51, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 4, "memoryGiB": 192, "name": "ml.g5.12xlarge", "vcpuNum": 48 }, { "_defaultOrder": 52, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 4, "memoryGiB": 384, "name": "ml.g5.24xlarge", "vcpuNum": 96 }, { "_defaultOrder": 53, "_isFastLaunch": false, "category": "Accelerated computing", "gpuNum": 8, "memoryGiB": 768, "name": "ml.g5.48xlarge", "vcpuNum": 192 } ], "kernelspec": { "display_name": "Python 3 (Data Science)", "language": "python", "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" } }, "nbformat": 4, "nbformat_minor": 5 }