{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geosemantics with Amazon Comprehend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import uuid\n",
    "\n",
    "comprehend = boto3.client(\"comprehend\")\n",
    "role = \"arn:aws:iam::141317253884:role/service-role/AmazonComprehendServiceRole-comprehend\"\n",
    "\n",
    "bucket = \"personalizelab-chicago\"\n",
    "entity_types = \"CHRONOSTRAT\"\n",
    "train_documents = \"bgs-geo-training-data.txt\"\n",
    "test_documents = \"bgs-geo-testing-data.txt\"\n",
    "entity_list = \"bgs-geo-entity-list.txt\"\n",
    "files = [train_documents, test_documents, entity_list]\n",
    "\n",
    "s3 = boto3.resource('s3')\n",
    "[s3.Bucket(bucket).upload_file(file, str(file)) for file in files]\n",
    "\n",
    "response = comprehend.create_entity_recognizer(\n",
    "    RecognizerName=\"geo-entity-{}\".format(str(uuid.uuid4())),\n",
    "    LanguageCode=\"en\",\n",
    "    DataAccessRoleArn= role,\n",
    "    InputDataConfig={\n",
    "        \"EntityTypes\": [\n",
    "            {\n",
    "                \"Type\": entity_types\n",
    "            }\n",
    "        ],\n",
    "        \"Documents\": {\n",
    "            \"S3Uri\": '/'.join(['s3:/', bucket, train_documents])\n",
    "        },\n",
    "        \"EntityList\": {\n",
    "            \"S3Uri\": '/'.join(['s3:/', bucket, entity_list])\n",
    "        }\n",
    "    }\n",
    ")\n",
    "recognizer_arn = response[\"EntityRecognizerArn\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional test to see the Entity Recognizer status\n",
    "\n",
    "import time\n",
    "\n",
    "while True:\n",
    "    response = comprehend.describe_entity_recognizer(\n",
    "        EntityRecognizerArn=recognizer_arn\n",
    "    )\n",
    "\n",
    "    status = response[\"EntityRecognizerProperties\"][\"Status\"]\n",
    "    if \"IN_ERROR\" == status:\n",
    "        sys.exit(1)\n",
    "    if \"TRAINED\" == status:\n",
    "        break\n",
    "\n",
    "    time.sleep(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'MessageId': '2993eb48-524a-5dd7-bb1f-bcce24fb7670',\n",
       " 'ResponseMetadata': {'RequestId': '1ac2bb7a-09be-5e4f-9cf1-ecfc7f0fd5e8',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '1ac2bb7a-09be-5e4f-9cf1-ecfc7f0fd5e8',\n",
       "   'content-type': 'text/xml',\n",
       "   'content-length': '294',\n",
       "   'date': 'Mon, 13 Jul 2020 14:49:21 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Optional code to send a text message once the training is complete\n",
    "\n",
    "phone_number = \"+12815159927\" # number for the scientist. Must include the international code (\"+1\" for the US)\n",
    "\n",
    "# Create an SNS client\n",
    "sns = boto3.client(\"sns\")\n",
    "\n",
    "sns.publish(\n",
    "    PhoneNumber = phone_number,\n",
    "    Message = \"{} training has stopped with status {}\".format(response[\"RecognizerName\"], status)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = comprehend.start_entities_detection_job(\n",
    "    EntityRecognizerArn=recognizer_arn,\n",
    "    JobName=\"Detection-Job-Name-{}\".format(str(uuid.uuid4())),\n",
    "    LanguageCode=\"en\",\n",
    "    DataAccessRoleArn=role,\n",
    "    InputDataConfig={\n",
    "        \"InputFormat\": \"ONE_DOC_PER_LINE\",\n",
    "        \"S3Uri\": '/'.join(['s3:/', bucket, test_documents])\n",
    "    },\n",
    "    OutputDataConfig={\n",
    "        \"S3Uri\": '/'.join(['s3:/', bucket, \"output\"])\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional code to send a text message once the detection job is complete\n",
    "\n",
    "while True:\n",
    "    response = comprehend.describe_entities_detection_job(\n",
    "        EntityRecognizerArn=recognizer_arn\n",
    "    )\n",
    "\n",
    "    status = response[\"EntitiesDetectionJobProperties\"][\"Status\"]\n",
    "    if \"IN_ERROR\" == status:\n",
    "        sys.exit(1)\n",
    "    if \"COMPLETED\" == status:\n",
    "        break\n",
    "\n",
    "    time.sleep(60)\n",
    "    \n",
    "sns.publish(\n",
    "    PhoneNumber = phone_number,\n",
    "    Message = \"{} job has stopped with status {}\".format(response[\"JobName\"], response[\"EntitiesDetectionJobProperties\"][\"JobStatus\"])\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}