{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e",
   "metadata": {},
   "source": [
    "# BasicTick: Create Cluster: HDB for basictickdb\n",
    "This notebook will create (start) an HDB cluster on a named basictick database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83c6572a-0972-4867-8c02-ea2c6c98427a",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<style>\n",
    "table {float:left}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f17a02c0-4f56-455c-a28a-dd102a88201c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Setup\n",
    "\n",
    "### Node Types\n",
    "|Type|Mem (GB)|vCPUs|\n",
    "|:---|---:|---:|\n",
    "|kx.s.large|12|2|\n",
    "|kx.s.xlarge|27|4|\n",
    "|kx.s.2xlarge|54|8|\n",
    "|kx.s.4xlarge|108|16|\n",
    "|kx.s.8xlarge|216|32|\n",
    "|kx.s.16xlarge|432|64|\n",
    "|kx.s.32xlarge|864|128|\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import json\n",
    "import datetime\n",
    "\n",
    "from managed_kx import *\n",
    "from env_kdb_1 import *\n",
    "\n",
    "# Cluster names and database\n",
    "from basictick_setup import *\n",
    "\n",
    "SEC_THREADS='4'\n",
    "\n",
    "BASE_NAME=\"HDB\"\n",
    "\n",
    "# Cluster Settings\n",
    "CODEBASE=\"basictick\"\n",
    "S3_CODE_PATH=\"code\"\n",
    "\n",
    "DB_PATHS = [ '/' ]\n",
    "\n",
    "CAPACITY_CONFIG={ 'nodeCount': 2, 'nodeType': 'kx.s.xlarge'}\n",
    "DATABASE_CONFIG=[{ 'databaseName': DB_NAME,'cacheConfigurations': [{'dbPaths': DB_PATHS, 'cacheType': 'CACHE_1000' }] }]\n",
    "CACHE_CONFIG=[{'type': 'CACHE_1000', 'size':1200}]\n",
    "\n",
    "CODE_CONFIG={ 's3Bucket': S3_BUCKET, 's3Key': f'{S3_CODE_PATH}/{CODEBASE}.zip' }\n",
    "INIT_SCRIPT=f'{CODEBASE}/hdbmkdb.q'\n",
    "\n",
    "CMD_ARGS=[\n",
    "    { 'key': 's', 'value': SEC_THREADS }, \n",
    "    { 'key': 'dbname', 'value': DB_NAME}, \n",
    "    { 'key': 'codebase', 'value': CODEBASE} \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# triggers credential get\n",
    "session=None\n",
    "\n",
    "try:\n",
    "    # aws: use ada for credentials\n",
    "    subprocess.call([\"which\", \"ada\"])\n",
    "    os.system(f\"ada credentials update --account={ACCOUNT_ID} --provider=isengard --role=Admin --once\")\n",
    "except: \n",
    "    None\n",
    "\n",
    "if AWS_ACCESS_KEY_ID is None:\n",
    "    print(\"Using Defaults ...\")\n",
    "    # create AWS session: using access variables\n",
    "    session = boto3.Session()\n",
    "else:\n",
    "    print(\"Using variables ...\")\n",
    "    session = boto3.Session(\n",
    "        aws_access_key_id=AWS_ACCESS_KEY_ID,\n",
    "        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n",
    "        aws_session_token=AWS_SESSION_TOKEN\n",
    "    )\n",
    "\n",
    "# create finspace client\n",
    "client = session.client(service_name='finspace', endpoint_url=ENDPOINT_URL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cd63f1e-0f36-410d-ab75-95fc2031d221",
   "metadata": {},
   "source": [
    "## Check Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9266a73-d208-4c6a-a50e-9c5b77e99704",
   "metadata": {},
   "outputs": [],
   "source": [
    "note_str = \"\"\n",
    "\n",
    "try:\n",
    "    c_set_list = client.list_kx_changesets(environmentId=ENV_ID, databaseName=DB_NAME)['kxChangesets']\n",
    "except:\n",
    "    note_str = \"<<Could not get changesets>>\"\n",
    "\n",
    "print(100*\"=\")\n",
    "print(f\"Database: {DB_NAME}, Changesets: {len(c_set_list)} {note_str}\")\n",
    "print(100*\"=\")\n",
    "\n",
    "# sort by create time\n",
    "c_set_list = sorted(c_set_list, key=lambda d: d['createdTimestamp']) \n",
    "\n",
    "for c in c_set_list:\n",
    "    c_set_id = c['changesetId']\n",
    "    print(f\"Changeset ({c['status']}): {c_set_id}: Created: {c['createdTimestamp']}\")\n",
    "    c_rqs = client.get_kx_changeset(environmentId=ENV_ID, databaseName=DB_NAME, changesetId=c_set_id)['changeRequests']\n",
    "\n",
    "    chs_pdf = pd.DataFrame.from_dict(c_rqs).style.hide(axis='index')\n",
    "    display(chs_pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b53b79fd-bbdc-45db-89a0-5c999951d47f",
   "metadata": {},
   "source": [
    "# Deploy code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dde345ef-524e-4f47-8ed4-f6e85f94e42c",
   "metadata": {},
   "outputs": [],
   "source": [
    "if AWS_ACCESS_KEY_ID is not None:\n",
    "    cp = f\"\"\"\n",
    "export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}\n",
    "export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}\n",
    "export AWS_SESSION_TOKEN={AWS_SESSION_TOKEN}\n",
    "\n",
    "zip -r -X {CODEBASE}.zip {CODEBASE} -x '*.ipynb_checkpoints*'       \n",
    "aws s3 cp {CODEBASE}.zip s3://{S3_BUCKET}/{S3_CODE_PATH}/{CODEBASE}.zip\n",
    "aws s3 ls {S3_BUCKET}/{S3_CODE_PATH}/\n",
    "\"\"\"\n",
    "else:\n",
    "    cp = f\"\"\"\n",
    "zip -r -X {CODEBASE}.zip {CODEBASE} -x '*.ipynb_checkpoints*'   \n",
    "aws s3 cp {CODEBASE}.zip s3://{S3_BUCKET}/{S3_CODE_PATH}/{CODEBASE}.zip\n",
    "aws s3 ls {S3_BUCKET}/{S3_CODE_PATH}/\n",
    "\"\"\"\n",
    "    \n",
    "# execute the S3 copy\n",
    "os.system(cp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67476efe-d308-4158-9e24-8fbe71509f76",
   "metadata": {},
   "source": [
    "## Create Cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e39ebf3-6940-40f1-a7f8-90efb3846f7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resp = client.create_kx_cluster(\n",
    "    environmentId=ENV_ID, \n",
    "    clusterName=HDB_CLUSTER_NAME,\n",
    "    clusterType='HDB',\n",
    "    releaseLabel = '1.0',\n",
    "    capacityConfiguration=CAPACITY_CONFIG,\n",
    "    databases=DATABASE_CONFIG,\n",
    "    cacheStorageConfigurations=CACHE_CONFIG,\n",
    "    clusterDescription=\"Created with create_HDB for basic_tick notebook\",\n",
    "    code=CODE_CONFIG,\n",
    "    initializationScript=INIT_SCRIPT,\n",
    "    commandLineArguments=CMD_ARGS,\n",
    "    azMode=AZ_MODE,\n",
    "    availabilityZoneId=AZ_ID,\n",
    "    vpcConfiguration={ \n",
    "        'vpcId': VPC_ID,\n",
    "        'securityGroupIds': SECURITY_GROUPS,\n",
    "        'subnetIds': SUBNET_IDS,\n",
    "        'ipAddressType': 'IP_V4' }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c23f37e1-9376-44b8-8e24-84ea5603b5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9946083-c9bb-4ecc-afbd-8c20e284ddf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=HDB_CLUSTER_NAME)\n",
    "print(\"** DONE **\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c50c578-05e8-49e7-8deb-1f6b94b10221",
   "metadata": {},
   "outputs": [],
   "source": [
    "cdf = get_clusters(client, environmentId=ENV_ID, clusterType=\"HDB\")\n",
    "\n",
    "display(cdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86f33240-bb12-49f3-8d9c-5783c25eb182",
   "metadata": {},
   "outputs": [],
   "source": [
    "print( f\"Last Run: {datetime.datetime.now()}\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d71daf7b-510b-4c0d-b285-360eb1c19533",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}