{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This Notebook creates a pandas dataframe to be used as training and validation data for XGBoost algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%` not found.\n"
     ]
    }
   ],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import csv\n",
    "import os\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import argparse\n",
    "import random\n",
    "import shutil\n",
    "import pandas as pd\n",
    "import glob\n",
    "import boto3\n",
    "import re\n",
    "from sagemaker import get_execution_role\n",
    "import PIL.Image\n",
    "import matplotlib.pyplot as plt\n",
    "from sagemaker.predictor import csv_serializer\n",
    "import io\n",
    "\n",
    "role = get_execution_role()\n",
    "region = boto3.Session().region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################\n",
    "## Get a folder with some files, create a list containing all \n",
    "#  file names\n",
    "##############################################################\n",
    "def get_filename_list(path):  \n",
    "    name = []\n",
    "    for path, subdirs, files in os.walk(path):\n",
    "        for filename in files:\n",
    "            if ('.JPG' in filename or '.png' in filename):\n",
    "                filename = filename.split('.')[0]\n",
    "                name.append(filename)\n",
    "    return name\n",
    "##############################################################\n",
    "## Get a folder with some files, create a list containing all \n",
    "#  paths to all files\n",
    "##############################################################\n",
    "def get_file_path_list(path):  \n",
    "    full_path_list = []\n",
    "    for path, subdirs, files in os.walk(path):\n",
    "        for filename in files:\n",
    "            f = os.path.join(path, filename)\n",
    "            full_path_list.append(f)\n",
    "    return full_path_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "img_path = 'path to your images'\n",
    "xml_path = 'path to your XML files'\n",
    "classes = ['Corroded','Clean']\n",
    "crop_images(xml_path, img_path,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Corroded = [p for p in files if \"Corroded\" in p]\n",
    "Clean = [p for p in files if \"Clean\" in p]\n",
    "\n",
    "print(\"Numebr of Corroded data points: \",len(Corroded))\n",
    "print(\"Numebr of Clean data points: \",len(Clean))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 2 - Put the cropped images in a single data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crop_path = 'Path to your cropped images'\n",
    "files = get_file_path_list(crop_path)\n",
    "\n",
    "cols = ['class','R','G','B']\n",
    "df = pd.DataFrame()\n",
    "\n",
    "classes_to_use = ['Corroded','Clean']\n",
    "dict1 = {'Clean': 0, 'Corroded': 1}\n",
    "for file in files:\n",
    "    lbls = Image.open(file)\n",
    "    imagenp = np.asarray(lbls)\n",
    "    imagenp=imagenp.reshape(imagenp.shape[1]*imagenp.shape[0],3)\n",
    "    name = file.split('\\\\')[-1].split('.')[0].split('-')[0]\n",
    "    classname = dict1[name]\n",
    "    dftemp = pd.DataFrame(imagenp)\n",
    "    dftemp.columns =['R','G','B']\n",
    "    dftemp['class'] = classname\n",
    "    columnsTitles=['class','R','G','B']\n",
    "    dftemp=dftemp.reindex(columns=columnsTitles)\n",
    "    df = pd.concat([df,dftemp], axis=0)\n",
    "\n",
    "df.columns = cols\n",
    "df.to_csv('data.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 3 -  Divide data into train and validation and upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = 'Your bucket name' # custom bucket name.\n",
    "prefix = 'csv'\n",
    "bucket_path = 'https://s3-{}.amazonaws.com/{}'.format(region,bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_split(FILE_DATA, FILE_TRAIN, FILE_VALIDATION, PERCENT_VALIDATION, TARGET_VAR):\n",
    "    data = pd.read_csv(FILE_DATA)\n",
    "    n = data.shape[0]\n",
    "    \n",
    "    # Make the first column the target feature    \n",
    "    cols = data.columns.tolist()\n",
    "    target_pos = data.columns.get_loc(TARGET_VAR)\n",
    "    cols.pop(target_pos)\n",
    "    cols = [TARGET_VAR] + cols\n",
    "    data = data.loc[:,cols]\n",
    "    \n",
    "    num_of_data = len(data)\n",
    "    num_train = int(((100-PERCENT_VALIDATION)/100.0)*n)\n",
    "    num_valid = int((PERCENT_VALIDATION/100.0)*n)\n",
    "        \n",
    "    # Shuffle the data\n",
    "    data = data.sample(frac=1, replace=False)\n",
    "    \n",
    "    # Split data\n",
    "    train_data = data.iloc[:num_train,:]\n",
    "    valid_data = data.iloc[(num_train+1):n,:]\n",
    "    \n",
    "    train_data.to_csv(FILE_TRAIN, index=False, header=False)\n",
    "    valid_data.to_csv(FILE_VALIDATION, index=False, header=False)\n",
    "    \n",
    "def write_to_s3(fobj, bucket, key):\n",
    "    return boto3.Session().resource('s3').Bucket(bucket).Object(key).upload_fileobj(fobj)\n",
    "\n",
    "def upload_to_s3(bucket, channel, filename):\n",
    "    fobj=open(filename, 'rb')\n",
    "    key = prefix+'/'+channel+'/'+filename\n",
    "    url = 's3://{}/{}'.format(bucket, key)\n",
    "    print('Writing to {}'.format(url))\n",
    "    write_to_s3(fobj, bucket, key)     \n",
    "    return(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_DATA = 'data.csv'\n",
    "TARGET_VAR = 'class'\n",
    "FILE_TRAIN = 'train.csv'\n",
    "FILE_VALIDATION = 'validation.csv'\n",
    "PERCENT_VALIDATION = 20\n",
    "\n",
    "data_split(FILE_DATA, FILE_TRAIN, FILE_VALIDATION, PERCENT_VALIDATION, TARGET_VAR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# upload the files to the S3 bucket\n",
    "s3_train_loc = upload_to_s3(bucket = bucket, channel = 'train', filename = FILE_TRAIN)\n",
    "s3_valid_loc = upload_to_s3(bucket = bucket, channel = 'validation', filename = FILE_VALIDATION)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}