{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Personalize with temporal evaluation on hold-out set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The notebook largely follows the basic notebook, with the additional tweak to hold-out 10% of \"future\" data for every user. Then, we set up an inference endpoint to bring recommendation and evaluate externally on the held-out data.\n",
    "\n",
    "The other minor difference is that we predict views rather than ratings. We find it more interesting to demonstrate recommendation of unpopular (yet highly personalized) movies than to recommend popular movies that everyone would enjoy (without personalization needs)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "from botocore.exceptions import ClientError\n",
    "!pip install tqdm\n",
    "from tqdm import tqdm_notebook\n",
    "from metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = str(np.random.uniform())[4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = \"demo-temporal-holdout-\"+   suffix        # replace with the name of your S3 bucket\n",
    "filename = \"DEMO-temporal-holdout.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 mb s3://{bucket}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize = boto3.client('personalize')\n",
    "personalize_runtime = boto3.client('personalize-runtime')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and process benchmark data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -N http://files.grouplens.org/datasets/movielens/ml-1m.zip\n",
    "!unzip -o ml-1m.zip\n",
    "data = pd.read_csv('./ml-1m/ratings.dat', sep='::', names=['USER_ID','ITEM_ID','EVENT_VALUE', 'TIMESTAMP'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[['USER_ID', 'ITEM_ID', 'TIMESTAMP']] # select columns that match the columns in the schema below\n",
    "print('unique users %d; unique items %d'%(\n",
    "    len(data['USER_ID'].unique()), len(data['ITEM_ID'].unique())))\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract last 10% of interactions per user as hold-out tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks = data.groupby('USER_ID').TIMESTAMP.rank(pct=True, method='first')\n",
    "data = data.join((ranks>0.9).to_frame('holdout'))\n",
    "holdout = data[data['holdout']].drop('holdout', axis=1)\n",
    "data = data[~data['holdout']].drop('holdout', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('unique users %d; unique items %d'%(\n",
    "    len(data['USER_ID'].unique()), len(data['ITEM_ID'].unique())))\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(filename, index=False)\n",
    "boto3.Session().resource('s3').Bucket(bucket).Object(filename).upload_file(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema = {\n",
    "    \"type\": \"record\",\n",
    "    \"name\": \"Interactions\",\n",
    "    \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"name\": \"USER_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"ITEM_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"TIMESTAMP\",\n",
    "            \"type\": \"long\"\n",
    "        }\n",
    "    ],\n",
    "    \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = \"DEMO-temporal-schema-\"+suffix,\n",
    "    schema = json.dumps(schema)\n",
    ")\n",
    "\n",
    "schema_arn = create_schema_response['schemaArn']\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets and Dataset Groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a Dataset Group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_dataset_group_response = personalize.create_dataset_group(\n",
    "    name = \"DEMO-temporal-dataset-group-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n",
    "print(json.dumps(create_dataset_group_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_group_response = personalize.describe_dataset_group(\n",
    "        datasetGroupArn = dataset_group_arn\n",
    "    )\n",
    "    status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n",
    "    print(\"DatasetGroup: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create an 'Interactions' Dataset Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_type = \"INTERACTIONS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = schema_arn,\n",
    "    name = \"DEMO-temporal-dataset-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## S3 Bucket Permissions for Personalize Access"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attach a Policy to the S3 Bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "policy = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"personalize.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": [\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"arn:aws:s3:::{}\".format(bucket),\n",
    "                \"arn:aws:s3:::{}/*\".format(bucket)\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a role that has the right permissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam = boto3.client(\"iam\")\n",
    "\n",
    "role_name = \"PersonalizeS3Role-\"+suffix\n",
    "assume_role_policy_document = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "          \"Effect\": \"Allow\",\n",
    "          \"Principal\": {\n",
    "            \"Service\": \"personalize.amazonaws.com\"\n",
    "          },\n",
    "          \"Action\": \"sts:AssumeRole\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "try:\n",
    "    create_role_response = iam.create_role(\n",
    "        RoleName = role_name,\n",
    "        AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n",
    "    );\n",
    "\n",
    "    iam.attach_role_policy(\n",
    "        RoleName = role_name,\n",
    "        PolicyArn = \"arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess\"\n",
    "    );\n",
    "\n",
    "    role_arn = create_role_response[\"Role\"][\"Arn\"]\n",
    "except ClientError as e:\n",
    "    if e.response['Error']['Code'] == 'EntityAlreadyExists':\n",
    "        role_arn = iam.get_role(RoleName=role_name)['Role']['Arn']\n",
    "    else:\n",
    "        raise\n",
    "# sometimes need to wait a bit for the role to be created\n",
    "time.sleep(45)\n",
    "print(role_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create your Dataset import jobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is your interactions data upload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"DEMO-temporal-dataset-import-job-\"+suffix,\n",
    "    datasetArn = dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, filename)\n",
    "    },\n",
    "    roleArn = role_arn\n",
    ")\n",
    "\n",
    "dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Dataset Import Job and Dataset Import Job Run to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_list = personalize.list_recipes()\n",
    "for recipe in recipe_list['recipes']:\n",
    "    print(recipe['recipeArn'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many recipes for different scenarios. In this example, we only have interactions data, so we will choose one from the basic recipes.\n",
    "\n",
    "| Feasible? | Recipe | Description \n",
    "|-------- | -------- |:------------\n",
    "| Y | aws-popularity-count | Calculates popularity of items based on count of events against that item in user-item interactions dataset.\n",
    "| Y | aws-hrnn | Predicts items a user will interact with. A hierarchical recurrent neural network which can model the temporal order of user-item interactions.\n",
    "| N - requires meta data | aws-hrnn-metadata | Predicts items a user will interact with. HRNN with additional features derived from contextual (user-item interaction metadata), user metadata (user dataset) and item metadata (item dataset)\n",
    "| N - for bandits and requires meta data | aws-hrnn-coldstart | Predicts items a user will interact with. HRNN-metadata with personalized exploration of new items.\n",
    "| N - for item-based queries | aws-sims | Computes items similar to a given item based on co-occurrence of item in same user history in user-item interaction dataset\n",
    "| N - for reranking a short list | aws-personalized-ranking | Reranks a list of items for a user. Trains on user-item interactions dataset. \n",
    "\n",
    "\n",
    "We (or autoML) can run all of these basic recipes and choose the best-performing model from internal metrics. We recommend comparisons, especially with popularity-baseline, to see the lifts in metrics via personalization. However, in this demo, we will pick one recipe - aws-hrnn, to focus on external evaluations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_arn = \"arn:aws:personalize:::recipe/aws-hrnn\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create and Wait for your Solution\n",
    "This is a 2 step process\n",
    "1. Create a Solution\n",
    "2. Create a Solution Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name = \"DEMO-temporal-solution-\"+suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn = recipe_arn,\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Solution Version to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Metrics of Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_metrics_response = personalize.get_solution_metrics(\n",
    "    solutionVersionArn = solution_version_arn\n",
    ")\n",
    "\n",
    "print(json.dumps(get_metrics_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Wait for Campaign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_campaign_response = personalize.create_campaign(\n",
    "    name = \"DEMO-temporal-campaign-\"+suffix,\n",
    "    solutionVersionArn = solution_version_arn,\n",
    "    minProvisionedTPS = 2,    \n",
    ")\n",
    "\n",
    "campaign_arn = create_campaign_response['campaignArn']\n",
    "print(json.dumps(create_campaign_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Campaign to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_campaign_response = personalize.describe_campaign(\n",
    "        campaignArn = campaign_arn\n",
    "    )\n",
    "    status = describe_campaign_response[\"campaign\"][\"status\"]\n",
    "    print(\"Campaign: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate using external metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An explanation of the evaluation metrics are provided at https://docs.aws.amazon.com/personalize/latest/dg/working-with-training-metrics.html\n",
    "\n",
    "For example, suppose we recommend four items and two of them are relevant, $r=[0,1,0,1]$. In this case, the metrics are:\n",
    "\n",
    "|Name\t|Example\t|Explanation\n",
    "|:------|:----------|:----------\n",
    "|Precision@K\t|$\\frac{2}{4} = 0.5$\t|Total relevant items divided by total recommended items.\n",
    "|Mean reciprocal ranks (MRR@K)\t|${\\rm mean}(\\frac{1}{2} + \\frac{1}{4}) = 0.375$\t|Considers positional effects by computing the mean of the inverse positions of all relevant items.\n",
    "|Normalized discounted cumulative gains (NDCG@K)\t|$\\frac{\\frac{1}{\\log(1 + 2)} + \\frac{1}{\\log(1 + 4)}}{\\frac{1}{\\log(1 + 1)} + \\frac{1}{\\log(1 + 2)}} = 0.65$\t|Considers positional effects by applying inverse logarithmic weights based on the positions of relevant items, normalized by the largest possible scores from ideal recommendations.\n",
    "|Average precision (AP@K)\t|${\\rm mean}(\\frac{1}{2} + \\frac{2}{4}) = 0.5$\t|Average precision@K where K is the position of every relevant item."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These metrics are different from the internal metrics in two aspects:\n",
    "* They are evaluated at different times, which may imply different click rates. We recommend to always keep the evaluations in the same time periods to avoid temporal drifts.\n",
    "* The example external evaluations may hold out and consider multiple items as ground truth, while the internal evaluations only hold out the last item in each user-history as the ground truth. There is no absolute preference as to how many items should be held out; we recommend designing the evaluation methods that are similar to the actual use case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "relevance = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user_id)\n",
    "    )\n",
    "    rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "    relevance.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in relevance]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in relevance]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in relevance]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in relevance]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optional: slightly better results after deduplicating previous purchase histories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel_dedup = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user_id)\n",
    "    )\n",
    "    past_items = data[data.USER_ID == user_id].ITEM_ID.values\n",
    "    topk = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "    rec_items = [x for x in topk if x not in past_items]\n",
    "    if len(rec_items) < 25:\n",
    "        rec_items.extend([x for x in topk if x not in rec_items])\n",
    "    rec_items = rec_items[:25]    \n",
    "\n",
    "    rel_dedup.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_dedup]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_dedup]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_dedup]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_dedup]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try comparing with popularity baseline as a dummy recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "topk = data.groupby(\"ITEM_ID\").TIMESTAMP.count().sort_values(ascending=False).iloc[:100].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel_popular = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_items = topk[:25]\n",
    "    rel_popular.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_popular]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_popular]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_popular]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_popular]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Popularity baseline deduplicating user histories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel_pop_dedup = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    past_items = data[data.USER_ID == user_id].ITEM_ID.values\n",
    "    rec_items = [x for x in topk if x not in past_items]\n",
    "    if len(rec_items) < 25:\n",
    "        rec_items.extend([x for x in topk if x not in rec_items])\n",
    "    rec_items = rec_items[:25]    \n",
    "    rel_pop_dedup.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_pop_dedup]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_pop_dedup]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_pop_dedup]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_pop_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in rel_pop_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_pop_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_pop_dedup]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize.delete_campaign(campaignArn=campaign_arn)\n",
    "while len(personalize.list_campaigns(solutionArn=solution_arn)['campaigns']):\n",
    "    time.sleep(5)\n",
    "\n",
    "personalize.delete_solution(solutionArn=solution_arn)\n",
    "while len(personalize.list_solutions(datasetGroupArn=dataset_group_arn)['solutions']):\n",
    "    time.sleep(5)\n",
    "\n",
    "for dataset in personalize.list_datasets(datasetGroupArn=dataset_group_arn)['datasets']:\n",
    "    personalize.delete_dataset(datasetArn=dataset['datasetArn'])\n",
    "while len(personalize.list_datasets(datasetGroupArn=dataset_group_arn)['datasets']):\n",
    "    time.sleep(5)\n",
    "\n",
    "personalize.delete_dataset_group(datasetGroupArn=dataset_group_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# If you are using a personal bucket Execute this cell with caution!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 rm s3://{bucket} --recursive"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}