{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4ceb1e53",
   "metadata": {},
   "source": [
    "# Model Registry 최신 모델을 이용한 모델 평가하기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afe7354b",
   "metadata": {},
   "source": [
    "## 1. 사전 준비 과정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dbea4a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "from sagemaker import get_execution_role\n",
    "from time import strftime\n",
    "import calendar\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85ee7180",
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e3f9781",
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_client = boto3.client('iam')\n",
    "role=get_execution_role()\n",
    "base_role_name=role.split('/')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e79fdb16",
   "metadata": {},
   "outputs": [],
   "source": [
    "sts_client = boto3.client(\"sts\")\n",
    "account_id = sts_client.get_caller_identity()['Account']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35c7e87c",
   "metadata": {},
   "source": [
    "### 1.1 MLOps에서 활용할 Policy 설정하기\n",
    "\n",
    "해당 HOL에서 구현할 아키텍처에 필요한 managed policy를 아래와 같이 정의합니다. Role을 별도 생성하셔도 되지만 HOL의 편의성을 위해 SageMaker Notebook/Studio와 동일한 Role에 policy를 추가하여 계속 활용합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9caf85a",
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_client.attach_role_policy(\n",
    "    RoleName=base_role_name,\n",
    "    PolicyArn='arn:aws:iam::aws:policy/AmazonEventBridgeFullAccess'\n",
    ")\n",
    "iam_client.attach_role_policy(\n",
    "    RoleName=base_role_name,\n",
    "    PolicyArn='arn:aws:iam::aws:policy/AWSLambda_FullAccess'\n",
    ")\n",
    "iam_client.attach_role_policy(\n",
    "    RoleName=base_role_name,\n",
    "    PolicyArn='arn:aws:iam::aws:policy/AWSCodeCommitFullAccess'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4407e77e",
   "metadata": {},
   "source": [
    "## 2. SageMaker Studio의 설정값에서 model_package_group_name 생성/가져오기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0598588f",
   "metadata": {},
   "source": [
    "model_package_groups_list = sm_client.list_model_package_groups(\n",
    "        SortBy='CreationTime',\n",
    "        SortOrder='Descending')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57926763",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_client = boto3.client('sagemaker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f4b6902",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_package_groups_list = sm_client.list_model_package_groups(\n",
    "        SortBy='CreationTime',\n",
    "        SortOrder='Descending')\n",
    "\n",
    "model_package_group_name_list = [model_package_group_name['ModelPackageGroupName'] for model_package_group_name \n",
    " in model_package_groups_list['ModelPackageGroupSummaryList']]\n",
    "\n",
    "model_package_group_name = \"yolov5\"\n",
    "\n",
    "\n",
    "if model_package_group_name not in model_package_group_name_list:\n",
    "    print(\"Creating a model package group for a new model\")\n",
    "\n",
    "    import time\n",
    "    model_package_group_input_dict = {\n",
    "     \"ModelPackageGroupName\" : model_package_group_name,\n",
    "     \"ModelPackageGroupDescription\" : \"Sample model package group\"\n",
    "    }\n",
    "\n",
    "    create_model_pacakge_group_response = sm_client.create_model_package_group(**model_package_group_input_dict)\n",
    "    print('ModelPackageGroup Arn : {}'.format(create_model_pacakge_group_response['ModelPackageGroupArn']))\n",
    "    print(f'model_package_group_name : {model_package_group_name}')\n",
    "else:    \n",
    "    for gname in model_package_groups_list['ModelPackageGroupSummaryList']:\n",
    "        if gname['ModelPackageGroupName'] == model_package_group_name:\n",
    "            model_package_group_name = gname['ModelPackageGroupName']\n",
    "            print(f'model_package_group_name : {model_package_group_name}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea047131",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sm_client.delete_model_package_group(ModelPackageGroupName=model_package_group_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c66f6c14",
   "metadata": {},
   "source": [
    "## 3. Create Amazon EventBridge Rule\n",
    "\n",
    "model registry에서 모델이 **Approved**되었을 때 이벤트 트리거를 만들기 위한 설정을 Amazon EventBridge Rule을 이용하여 설정합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65332fc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "event_client = boto3.client('events')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0190750",
   "metadata": {},
   "outputs": [],
   "source": [
    "eventpattern = json.dumps(\n",
    "    {\n",
    "      \"source\": [\"aws.sagemaker\"],\n",
    "      \"detail-type\": [\"SageMaker Model Package State Change\"],\n",
    "      \"detail\": {\n",
    "        \"ModelPackageGroupName\": [f\"{model_package_group_name}\"],\n",
    "        \"ModelApprovalStatus\": [\"Approved\"]\n",
    "      }\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "057fd198",
   "metadata": {},
   "outputs": [],
   "source": [
    "rule_name = 'yolov5_model_package_state'\n",
    "event_rule = event_client.put_rule(\n",
    "    Name=rule_name,\n",
    "    EventPattern=eventpattern,\n",
    "    State='ENABLED',\n",
    "    Description='This is after the approval update for the yolov5 model',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa045373",
   "metadata": {},
   "source": [
    "## 4. 모델 평가를 위한 Lambda function 생성\n",
    "\n",
    "EventBridge 에서 Rule 만족하는 이벤트가 발생했을 때 실행되는 Lambda Function을 정의합니다. Lambda Function 은 테스트 데이터를 예측하는 Batch transform job을 수행하게 됩니다.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f17ba22d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_client = boto3.client('lambda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fda553b",
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_trust_policy=json.dumps({\n",
    "  \"Version\": \"2012-10-17\",\n",
    "  \"Statement\": [\n",
    "    {\n",
    "      \"Effect\": \"Allow\",\n",
    "      \"Principal\": {\n",
    "        \"Service\": \"lambda.amazonaws.com\"\n",
    "      },\n",
    "      \"Action\": \"sts:AssumeRole\"\n",
    "    }\n",
    "  ]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5966fa39",
   "metadata": {},
   "outputs": [],
   "source": [
    "role_name='lambda-assume-role_'+ strftime(\"%m%d-%H%M%s\")\n",
    "try:\n",
    "    for role_list in iam_client.list_roles()['Roles']:\n",
    "        pre_role_name = role_list['RoleName']\n",
    "        if pre_role_name.split(\"_\")[0] in ['lambda-assume-role']:\n",
    "            iam_client.detach_role_policy(\n",
    "                RoleName=pre_role_name,\n",
    "                PolicyArn='arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'\n",
    "            )\n",
    "            iam_client.detach_role_policy(\n",
    "                RoleName=pre_role_name,\n",
    "                PolicyArn='arn:aws:iam::aws:policy/AmazonSageMakerFullAccess'\n",
    "            )\n",
    "            iam_client.delete_role(RoleName=pre_role_name)\n",
    "except:\n",
    "    pass\n",
    "finally:\n",
    "    lambda_role = iam_client.create_role(\n",
    "        RoleName=role_name,\n",
    "        AssumeRolePolicyDocument=lambda_trust_policy\n",
    "    )\n",
    "    iam_client.attach_role_policy(\n",
    "        RoleName=role_name,\n",
    "        PolicyArn='arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'\n",
    "    )\n",
    "    iam_client.attach_role_policy(\n",
    "        RoleName=role_name,\n",
    "        PolicyArn='arn:aws:iam::aws:policy/AmazonSageMakerFullAccess'\n",
    "    )\n",
    "    iam_client.attach_role_policy(\n",
    "        RoleName=role_name,\n",
    "        PolicyArn='arn:aws:iam::aws:policy/SecretsManagerReadWrite'\n",
    "    )\n",
    "    time.sleep(10)\n",
    "role_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a167a682",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The name of our algorithm\n",
    "algo_name = \"yolov5-batch-inference\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6d0530b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "my_session = boto3.session.Session()\n",
    "region = my_session.region_name\n",
    "\n",
    "repo_name=f\"{account_id}.dkr.ecr.{region}.amazonaws.com/{algo_name}:1.0\"\n",
    "repo_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1c32114",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%capture\n",
    "!sh build_and_push.sh $account_id $region $algo_name $repo_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbf58c10",
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r\n",
    "print(f\"bucket : {bucket} \\n sec_arn : {sec_arn} \\ncodecommit_repo : {codecommit_repo}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79bf7dda",
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_name='yolov5-batch-inference'\n",
    "try:\n",
    "    lambda_client.delete_function(FunctionName=lambda_name)\n",
    "except:\n",
    "    pass\n",
    "finally:\n",
    "    lambda_response = lambda_client.create_function(\n",
    "        FunctionName=lambda_name,\n",
    "        Role=lambda_role['Role']['Arn'],\n",
    "        Code={\n",
    "            'ImageUri': repo_name\n",
    "        },\n",
    "        PackageType='Image',\n",
    "        Description='Create the latest version-based yolov5 model',\n",
    "        Timeout=600,\n",
    "        MemorySize=512,\n",
    "        Environment={\n",
    "          'Variables': {\n",
    "              \"bucket_name\" : bucket,\n",
    "              \"model_package_group_name\" : model_package_group_name,\n",
    "              \"sec_arn\" : sec_arn,\n",
    "              \"role\" : role,\n",
    "              \"instanace_type\" : \"ml.c5.2xlarge\",\n",
    "              \"instanace_count\" : \"1\",\n",
    "              \"region_name\" : region,\n",
    "              \"codecommit_repo\" : codecommit_repo\n",
    "              \n",
    "              \n",
    "          }\n",
    "      }\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c7cc826",
   "metadata": {},
   "outputs": [],
   "source": [
    "# response = lambda_client.update_function_code(\n",
    "#     FunctionName=lambda_name,\n",
    "#     ImageUri=repo_name\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4cb752e",
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_permission_response = lambda_client.add_permission(\n",
    "    FunctionName=lambda_name,\n",
    "    StatementId='InvokeLambdaFunction',\n",
    "    Action='lambda:InvokeFunction',\n",
    "    Principal=\"events.amazonaws.com\",\n",
    "    SourceArn=event_rule['RuleArn'],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ed81162",
   "metadata": {},
   "source": [
    "Amazon EventBridge에 위에서 생성한 Lambda function을 타켓으로 설정합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df1db8a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "event_client.put_targets(\n",
    "    Rule=rule_name,\n",
    "    Targets=[\n",
    "        {\n",
    "            'Id': 'Target0',\n",
    "            'Arn': lambda_response['FunctionArn']\n",
    "        }\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "029c50c7",
   "metadata": {},
   "source": [
    "## 5. TEST - Approval the latest Model Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fb03f7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"artifacts_dir : {artifacts_dir}, model_package_group_name : {model_package_group_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3052c819",
   "metadata": {},
   "outputs": [],
   "source": [
    "latest_modelpkg_list = sm_client.list_model_packages( ModelPackageGroupName=model_package_group_name,\n",
    "                                                      ModelApprovalStatus='PendingManualApproval',\n",
    "                                                      SortBy='CreationTime',\n",
    "                                                      SortOrder='Descending')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e90c267e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ModelPackageArn = latest_modelpkg_list['ModelPackageSummaryList'][0]['ModelPackageArn']\n",
    "ModelPackageArn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d5b0ee7",
   "metadata": {},
   "source": [
    "#### - 최신 배포된 모델에 대해 승인 수행"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51817813",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_package_update_input_dict = {\n",
    "    \"ModelPackageArn\" : ModelPackageArn,\n",
    "    \"ModelApprovalStatus\" : \"Approved\"\n",
    "}\n",
    "model_package_update_response = sm_client.update_model_package(**model_package_update_input_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecc3fc45",
   "metadata": {},
   "source": [
    "#### - Model Version이 필요한 경우 가상으로 추가하는 명령어"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3f736d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import sagemaker\n",
    "# image_uri = sagemaker.image_uris.retrieve(framework='pytorch', \n",
    "#                                               image_scope='training',\n",
    "#                                               version='1.10',\n",
    "#                                               instance_type='ml.c5.2xlarge', \n",
    "#                                               region=region)\n",
    "# print(image_uri)\n",
    "# modelpackage_inference_specification =  {\n",
    "#     \"InferenceSpecification\": {\n",
    "#       \"Containers\": [\n",
    "#          {\n",
    "#             \"Image\": image_uri,\n",
    "#          }\n",
    "#       ],\n",
    "#       \"SupportedContentTypes\": [ \"application/x-image\" ],\n",
    "#       \"SupportedResponseMIMETypes\": [ \"application/x-image\" ],\n",
    "#    }\n",
    "#  }\n",
    "# # Specify the model source\n",
    "# model_url = f\"{artifacts_dir}model.tar.gz\"\n",
    "\n",
    "# # Specify the model data\n",
    "# modelpackage_inference_specification[\"InferenceSpecification\"][\"Containers\"][0][\"ModelDataUrl\"]=model_url\n",
    "\n",
    "# create_model_package_input_dict = {\n",
    "#     \"ModelPackageGroupName\" : model_package_group_name,\n",
    "#     \"ModelPackageDescription\" : \"Model to detect 3 different types ('Platelets', 'RBC', 'WBC')\",\n",
    "#     \"ModelApprovalStatus\" : \"PendingManualApproval\"\n",
    "# }\n",
    "# create_model_package_input_dict.update(modelpackage_inference_specification)\n",
    "\n",
    "# create_mode_package_response = sm_client.create_model_package(**create_model_package_input_dict)\n",
    "# model_package_arn = create_mode_package_response[\"ModelPackageArn\"]\n",
    "# print('ModelPackage Version ARN : {}'.format(model_package_arn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "465b82a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "409a7ef4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}