{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import autogluon as ag\n", "from autogluon.tabular import TabularDataset, TabularPredictor\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import warnings\n", "import collections\n", "from sklearn.model_selection import StratifiedKFold, train_test_split, ShuffleSplit\n", "from sklearn.metrics import (confusion_matrix, classification_report, accuracy_score, roc_curve, average_precision_score,\n", " precision_recall_curve, precision_score, recall_score, f1_score, matthews_corrcoef, auc)\n", "try:\n", " from joblib import dump, load\n", "except ImportError:\n", " from sklearn.externals.joblib import dump, load\n", "\n", " \n", "def plot_roc_curve(y_true, y_score, is_single_fig=False):\n", " \"\"\"\n", " Plot ROC Curve and show AUROC score\n", " \"\"\" \n", " fpr, tpr, _ = roc_curve(y_true, y_score)\n", " roc_auc = auc(fpr, tpr)\n", " plt.title('AUROC = {:.4f}'.format(roc_auc))\n", " plt.plot(fpr, tpr, 'b')\n", " plt.plot([0,1], [0,1], 'r--')\n", " plt.xlim([-0.05,1.05])\n", " plt.ylim([-0.05,1.05])\n", " plt.ylabel('TPR(True Positive Rate)')\n", " plt.xlabel('FPR(False Positive Rate)')\n", " if is_single_fig:\n", " plt.show()\n", " \n", "def plot_pr_curve(y_true, y_score, is_single_fig=False):\n", " \"\"\"\n", " Plot Precision Recall Curve and show AUPRC score\n", " \"\"\"\n", " prec, rec, thresh = precision_recall_curve(y_true, y_score)\n", " avg_prec = average_precision_score(y_true, y_score)\n", " plt.title('AUPRC = {:.4f}'.format(avg_prec))\n", " plt.step(rec, prec, color='b', alpha=0.2, where='post')\n", " plt.fill_between(rec, prec, step='post', alpha=0.2, color='b')\n", " plt.plot(rec, prec, 'b')\n", " plt.xlim([-0.05,1.05])\n", " plt.ylim([-0.05,1.05])\n", " plt.ylabel('Precision')\n", " plt.xlabel('Recall')\n", " if is_single_fig:\n", " plt.show()\n", "\n", "def plot_conf_mtx(y_true, y_score, thresh=0.5, class_labels=['0','1'], is_single_fig=False):\n", " \"\"\"\n", " Plot Confusion matrix\n", " \"\"\" \n", " y_pred = np.where(y_score >= thresh, 1, 0)\n", " print(\"confusion matrix (cutoff={})\".format(thresh))\n", " print(classification_report(y_true, y_pred, target_names=class_labels))\n", " conf_mtx = confusion_matrix(y_true, y_pred)\n", " sns.heatmap(conf_mtx, xticklabels=class_labels, yticklabels=class_labels, annot=True, fmt='d')\n", " plt.title('Confusion Matrix')\n", " plt.ylabel('True Class')\n", " plt.xlabel('Predicted Class')\n", " if is_single_fig:\n", " plt.show()\n", " \n", "def prob_barplot(y_score, bins=np.arange(0.0, 1.11, 0.1), right=False, filename=None, figsize=(10,4), is_single_fig=False):\n", " \"\"\"\n", " Plot barplot by binning predicted scores ranging from 0 to 1\n", " \"\"\" \n", " c = pd.cut(y_score, bins, right=right)\n", " counts = c.value_counts()\n", " percents = 100. * counts / len(c)\n", " percents.plot.bar(rot=0, figsize=figsize)\n", " plt.title('Histogram of score')\n", " print(percents)\n", " if filename is not None:\n", " plt.savefig('{}.png'.format(filename)) \n", " if is_single_fig:\n", " plt.show()\n", " \n", "def show_evals(y_true, y_score, thresh=0.5):\n", " \"\"\"\n", " All-in-one function for evaluation. \n", " \"\"\" \n", " plt.figure(figsize=(14,4))\n", " plt.subplot(1,3,1)\n", " plot_roc_curve(y_true, y_score)\n", " plt.subplot(1,3,2) \n", " plot_pr_curve(y_true, y_score)\n", " plt.subplot(1,3,3) \n", " plot_conf_mtx(y_true, y_score, thresh) \n", " plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "MODEL_DIR = 'model_pkg'\n", "S3_MODEL_PATH = '[YOUR S3 MODEL PATH]' # YOUR S3 MODEL PATH\n", "TEST_FILE = 'test.csv' # YOUR TEST CSV FILE\n", "LABEL_COLUMN = 'label' # YOUR TARGET COLUMN\n", "THRESH = 0.5" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!aws s3 cp {S3_MODEL_PATH} .\n", "!rm -rf {MODEL_DIR}\n", "!mkdir {MODEL_DIR}\n", "!tar -xzvf model.tar.gz -C {MODEL_DIR}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_data = TabularDataset(file_path=TEST_FILE)\n", "y_test = test_data[LABEL_COLUMN] # values to predict\n", "test_data_nolab = test_data.drop(labels=[LABEL_COLUMN],axis=1) # delete label column to prove we're not cheating" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "predictor = TabularPredictor(MODEL_DIR) \n", "y_prob = predictor.predict_proba(test_data_nolab)\n", "y_pred = y_prob > THRESH" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "perf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "perf[\"confusion_matrix\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results = predictor.fit_summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "show_evals(y_test, y_prob)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "\n", "# Feature importance\n", "featimp = predictor.feature_importance(test_data)\n", "fig, ax = plt.subplots(figsize=(12,5))\n", "plot = sns.barplot(x=featimp.index, y=featimp.values)\n", "ax.set_title('Feature Importance')\n", "plot.set_xticklabels(plot.get_xticklabels(), rotation='vertical')\n", "\n", "plt.show()" ] } ], "metadata": { "kernelspec": { "display_name": "conda_mxnet_p37", "language": "python", "name": "conda_mxnet_p37" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" } }, "nbformat": 4, "nbformat_minor": 4 }