{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run init_model.py 'algo_ml_long_short_predict'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1) Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get S3 bucket\n",
    "s3bucket=!(aws s3 ls | grep algotrading- | awk  '{print $3}')\n",
    "s3bucket=s3bucket[0]\n",
    "s3bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install PyAthena"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sagemaker as sage\n",
    "from sagemaker import get_execution_role\n",
    "import datetime\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "import json\n",
    "\n",
    "role = get_execution_role()\n",
    "sess = sage.Session()\n",
    "region = sess.boto_session.region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pyathena import connect\n",
    "conn = connect(s3_staging_dir='s3://'+s3bucket+'/results/',\n",
    "               region_name=region)\n",
    "\n",
    "df = pd.read_sql(\"SELECT dt,open,high,low,close,vol FROM algo_data.hist_data_daily;\", conn)\n",
    "df.set_index(pd.DatetimeIndex(df['dt']),inplace=True)\n",
    "del df['dt']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainCount=int(len(df)*0.4)\n",
    "dfTrain = df.iloc[:trainCount]\n",
    "\n",
    "dfTest = df.iloc[trainCount:]\n",
    "dfTest.to_csv('local/'+algo_name+'/input/data/training/data.csv')\n",
    "dfTest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "dfTest[\"close\"].plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2) Modify Strategy Configuration \n",
    "\n",
    "In the following cell, you can adjust the parameters for the strategy.\n",
    "\n",
    "* `user` = Name for Leaderboard (optional)\n",
    "* `long_threshold` = Threshold for Long Trade (0 to 1)\n",
    "* `short_threshold` = Threshold for Short Trade (0 to 1)\n",
    "* `profit_target_pct` = Profit Target Percentage \n",
    "* `stop_target_pct` = Stop Target Percentage\n",
    "* `size` = The number of shares for a transaction\n",
    "\n",
    "`Tip`: A good starting point for improving the strategy is modify the profit / stop target and the risk/reward ratio. Another option is to reduce the number of signals by increasing the threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile local/{algo_name}/input/config/hyperparameters.json\n",
    "{ \"user\" : \"user\",\n",
    "  \"long_threshold\" : \"0.5\",\n",
    "  \"short_threshold\" : \"0.5\",\n",
    "  \"profit_target_pct\" : \"2.00\",\n",
    "  \"stop_target_pct\" : \"1.50\",\n",
    "  \"size\" : \"100\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run update_config.py $algo_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3) Modify Strategy Code\n",
    "\n",
    "In the following cell, you can modify the strategy code. For the first backtests, you can leave it as is.\n",
    "\n",
    "`Tip`: A good starting point for improving the strategy is to combine the signal from the model with traditional trend indicators (e.g. moving average). This will likely improve the performance. To improve the strategy further, you could increase the accuracy of the machine learning model by including more indicators (e.g. ATR) or modify the input and forecast window. This requires to re-train the machine learning model as this needs to match your strategy. For timeseries forecasting, you could compare the performance with more advanced ML networks (e.g. CNN, LTSM, RNN) and pick the model with the best predictions.\n",
    "\n",
    "You can also checkout other AWS-provided options for timeseries forecasting and formulate a strategy that uses price predictions and integrate them in your strategy:\n",
    "* https://docs.aws.amazon.com/sagemaker/latest/dg/deepar.html \n",
    "* https://aws.amazon.com/forecast/\n",
    "\n",
    "Here are some helpful links:\n",
    "* Backtrader Documentation: https://www.backtrader.com/docu/strategy/\n",
    "* TA-Lib Indicator Reference: https://www.backtrader.com/docu/talibindautoref/\n",
    "* Backtrader Indicator Reference: https://www.backtrader.com/docu/indautoref/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile model/{algo_name}.py\n",
    "import backtrader as bt\n",
    "from algo_base import *\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import load_model\n",
    "\n",
    "class MyStrategy(StrategyTemplate):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(MyStrategy, self).__init__()\n",
    "        self.config[\"long_threshold\"]=float(self.config[\"long_threshold\"])\n",
    "        self.config[\"short_threshold\"]=float(self.config[\"short_threshold\"])\n",
    "        self.config[\"size\"]=int(self.config[\"size\"])\n",
    "        self.config[\"profit_target_pct\"]=float(self.config[\"profit_target_pct\"])\n",
    "        self.config[\"stop_target_pct\"]=float(self.config[\"stop_target_pct\"])\n",
    "\n",
    "        self.order=None\n",
    "        self.orderPlaced=False\n",
    "                                \n",
    "        self.model = load_model('model_long_short_predict.h5')\n",
    "        \n",
    "        # input / indicators\n",
    "        self.repeatCount=15\n",
    "        self.repeatStep=1\n",
    "        \n",
    "        self.profitTarget=self.config[\"profit_target_pct\"]/100.0\n",
    "        self.stopTarget=self.config[\"stop_target_pct\"]/100.0\n",
    "        self.size=self.config[\"size\"]\n",
    "         \n",
    "        self.sma=[]\n",
    "        self.roc=[]\n",
    "        \n",
    "        self.hData=[\"dt\"]\n",
    "        self.hData.append(\"close\") \n",
    "        for a in range(0,self.repeatCount):\n",
    "            tp=(a+1)*self.repeatStep+1\n",
    "            self.hData.append(\"sma\"+str(tp))\n",
    "            self.sma.append(bt.talib.SMA(self.data, timeperiod=tp, plot=False))\n",
    "        for a in range(0,self.repeatCount):\n",
    "            tp=(a+1)*self.repeatStep+1\n",
    "            self.hData.append(\"roc\"+str(tp))\n",
    "            self.roc.append(bt.talib.ROC(self.data, timeperiod=tp, plot=False))\n",
    "\n",
    "    def init_broker(broker):\n",
    "        broker.setcash(100000.0)\n",
    "        broker.setcommission(commission=0.0) \n",
    "        \n",
    "    def add_data(cerebro):\n",
    "        data = btfeeds.GenericCSVData(\n",
    "            dataname=MyStrategy.TRAIN_FILE,\n",
    "            dtformat=('%Y-%m-%d'),\n",
    "            timeframe=bt.TimeFrame.Days,\n",
    "            datetime=0,\n",
    "            time=-1,\n",
    "            high=2,\n",
    "            low=3,\n",
    "            open=1,\n",
    "            close=4,\n",
    "            volume=5,\n",
    "            openinterest=-1\n",
    "        )\n",
    "        cerebro.adddata(data)\n",
    "\n",
    "    def next(self):\n",
    "        super(MyStrategy, self).next()\n",
    "        \n",
    "        dt=self.datas[0].datetime.datetime(0)\n",
    "        cl=self.dataclose[0]\n",
    "        inputRec=[]                \n",
    "\n",
    "        #open\n",
    "        inputRec0=[]\n",
    "        inputRec0.append(cl)\n",
    "\n",
    "        #sma\n",
    "        for a in range(0,self.repeatCount):\n",
    "            if math.isnan(self.sma[a][0]):\n",
    "                inputRec0.append(cl)\n",
    "            else:\n",
    "                inputRec0.append(self.sma[a][0])\n",
    "\n",
    "        m1=min(inputRec0)\n",
    "        m2=max(inputRec0)\n",
    "        for a in inputRec0:\n",
    "            if m2-m1==0:\n",
    "                inputRec.append(0)\n",
    "            else:\n",
    "                inputRec.append((a-m1)/(m2-m1))\n",
    "\n",
    "        #roc\n",
    "        for a in range(0,self.repeatCount):\n",
    "            if math.isnan(self.roc[a][0]):\n",
    "                inputRec.append(0)\n",
    "            else:\n",
    "                inputRec.append(self.roc[a][0])\n",
    "\n",
    "        mX=[]\n",
    "        mX.append(np.array(inputRec))\n",
    "        dataX=np.array(mX)\n",
    "        #print(\"dataX=%s\" % dataX)\n",
    "\n",
    "        # *** ML prediction ***\n",
    "        mY=self.model.predict(dataX)\n",
    "        #print(\"mY=%s\" % mY)\n",
    "        tLong=mY[0][0]\n",
    "        tShort=mY[0][1]\n",
    "        #print(\"[%s]:long=%s,short=%s\" % (dt,tLong,tShort))\n",
    "        if not self.position:\n",
    "            fLong=(tLong>self.config[\"long_threshold\"]) \n",
    "            fShort=(tShort>self.config[\"short_threshold\"])\n",
    "            if fLong:\n",
    "                self.order=self.buy(size=self.size)\n",
    "                self.limitPrice=cl+self.profitTarget*cl\n",
    "                self.stopPrice=cl-self.stopTarget*cl\n",
    "            elif fShort:\n",
    "                self.order=self.sell(size=self.size)                    \n",
    "                self.limitPrice=cl-self.profitTarget*cl\n",
    "                self.stopPrice=cl+self.stopTarget*cl\n",
    "\n",
    "        if self.position:\n",
    "            if self.position.size>0:\n",
    "                if cl>=self.limitPrice or cl<=self.stopPrice:\n",
    "                    self.order=self.sell(size=self.size)\n",
    "            elif self.position.size<0:\n",
    "                if cl<=self.limitPrice or cl>=self.stopPrice:\n",
    "                    self.order=self.buy(size=self.size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 4) Backtest Locally\n",
    "\n",
    "**Please note that the initial docker build may take a few minutes. Subsequent runs are fast.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Build Local Algo Image\n",
    "!docker build -t $algo_name .\n",
    "!docker run -v $(pwd)/local/$algo_name:/opt/ml --rm $algo_name train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from IPython.display import Image\n",
    "Image(filename='local/'+algo_name+'/model/chart.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Refine your trading strategy (step 2 to 4). Once you are ready, move on to the next step."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 5) Backtest on SageMaker and submit performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Deploy Algo Image to ECS\n",
    "!./build_and_push.sh $algo_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Run Remote Forwardtest via SageMaker\n",
    "import sagemaker as sage\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.estimator import Estimator \n",
    "\n",
    "role = get_execution_role()\n",
    "sess = sage.Session()\n",
    "\n",
    "WORK_DIRECTORY = 'local/'+algo_name+'/input/data/training'\n",
    "data_location = sess.upload_data(WORK_DIRECTORY, key_prefix='data')\n",
    "print(data_location)\n",
    "\n",
    "conf_file='local/'+algo_name+'/input/config/hyperparameters.json'\n",
    "with open(conf_file, 'r') as f:\n",
    "    config = json.load(f)\n",
    "#config['sim_data']='True'\n",
    "print(config)\n",
    "\n",
    "prefix=algo_name\n",
    "job_name=prefix.replace('_','-')\n",
    "\n",
    "account = sess.boto_session.client('sts').get_caller_identity()['Account']\n",
    "region = sess.boto_session.region_name\n",
    "image = f'{account}.dkr.ecr.{region}.amazonaws.com/{prefix}:latest'\n",
    "\n",
    "algo = sage.estimator.Estimator(\n",
    "    image_uri=image,\n",
    "    role=role,\n",
    "    instance_count=1,\n",
    "    instance_type='ml.m4.xlarge',\n",
    "    output_path=\"s3://{}/output\".format(sess.default_bucket()),\n",
    "    sagemaker_session=sess,\n",
    "    base_job_name=job_name,\n",
    "    hyperparameters=config,\n",
    "    metric_definitions=[\n",
    "        {\n",
    "            \"Name\": \"algo:pnl\",\n",
    "            \"Regex\": \"Total PnL:(.*?)]\"\n",
    "        },\n",
    "        {\n",
    "            \"Name\": \"algo:sharpe_ratio\",\n",
    "            \"Regex\": \"Sharpe Ratio:(.*?),\"\n",
    "        }\n",
    "    ])\n",
    "algo.fit(data_location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get Algo Metrics\n",
    "from sagemaker.analytics import TrainingJobAnalytics\n",
    "\n",
    "latest_job_name = algo.latest_training_job.job_name\n",
    "metrics_dataframe = TrainingJobAnalytics(training_job_name=latest_job_name).dataframe()\n",
    "metrics_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Get Algo Chart from S3\n",
    "model_name=algo.model_data.replace('s3://'+sess.default_bucket()+'/','')\n",
    "import boto3\n",
    "s3 = boto3.resource('s3')\n",
    "my_bucket = s3.Bucket(sess.default_bucket())\n",
    "my_bucket.download_file(model_name,'model.tar.gz')\n",
    "!tar -xzf model.tar.gz\n",
    "!rm model.tar.gz\n",
    "from IPython.display import Image\n",
    "Image(filename='chart.png') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Congratulations! You've completed this strategy."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}