{ "cells": [ { "cell_type": "markdown", "id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e", "metadata": {}, "source": [ "# Create Cluster: RDB\n", "This notebook will create (start) an RDB cluster." ] }, { "cell_type": "code", "execution_count": 1, "id": "83c6572a-0972-4867-8c02-ea2c6c98427a", "metadata": { "jupyter": { "source_hidden": true }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%html\n", "" ] }, { "cell_type": "markdown", "id": "f17a02c0-4f56-455c-a28a-dd102a88201c", "metadata": { "tags": [] }, "source": [ "## Setup\n", "\n", "### Node Types\n", "|Type|Mem (GB)|vCPUs|\n", "|:---|---:|---:|\n", "|kx.s.large|12|2|\n", "|kx.s.xlarge|27|4|\n", "|kx.s.2xlarge|54|8|\n", "|kx.s.4xlarge|108|16|\n", "|kx.s.8xlarge|216|32|\n", "|kx.s.16xlarge|432|64|\n", "|kx.s.32xlarge|864|128|\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb", "metadata": {}, "outputs": [], "source": [ "import os\n", "import boto3\n", "import json\n", "import datetime\n", "\n", "from managed_kx import *\n", "from env_2 import *\n", "\n", "# Managed KX Database and Cluster names to create\n", "DB_NAME=\"welcomedb\"\n", "\n", "SEC_THREADS='8'\n", "CLUSTER_NAME=f\"RDB_{DB_NAME}\"\n", "\n", "# Cluster Settings\n", "CODEBASE=\"code\"\n", "S3_CODE_PATH=\"code\"\n", "\n", "CAPACITY_CONFIG={ 'nodeCount': 1, 'nodeType': 'kx.s.2xlarge'}\n", "SAVEDOWN_CONFIG={ 'type': 'SDS01', 'size':200 }\n", "DATABASE_CONFIG=[{ 'databaseName': DB_NAME }]\n", "CODE_CONFIG={ 's3Bucket': S3_BUCKET, 's3Key': f'{S3_CODE_PATH}/{CODEBASE}.zip' }\n", "\n", "INIT_SCRIPT=f'{CODEBASE}/init.q'\n", "CMD_ARGS=[\n", " { 'key': 's', 'value': SEC_THREADS }, \n", " { 'key': 'dbname', 'value': DB_NAME}, \n", " { 'key': 'codebase', 'value': CODEBASE} \n", "]" ] }, { "cell_type": "code", "execution_count": 3, "id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using variables ...\n" ] } ], "source": [ "# triggers credential get\n", "session=None\n", "\n", "try:\n", " subprocess.call([\"which\", \"ada\"])\n", " os.system(f\"ada credentials update --account={ACCOUNT_ID} --provider=isengard --role=Admin --once\")\n", "except: \n", " None\n", "\n", "if AWS_ACCESS_KEY_ID is None:\n", " print(\"Using Defaults ...\")\n", " # create AWS sessio: using access variables\n", " session = boto3.Session()\n", "else:\n", " print(\"Using variables ...\")\n", " session = boto3.Session(\n", " aws_access_key_id=AWS_ACCESS_KEY_ID,\n", " aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n", " aws_session_token=AWS_SESSION_TOKEN\n", " )\n", "\n", "# create finspace client\n", "client = session.client(service_name='finspace', endpoint_url=ENDPOINT_URL)" ] }, { "cell_type": "markdown", "id": "67476efe-d308-4158-9e24-8fbe71509f76", "metadata": {}, "source": [ "## Create Cluster" ] }, { "cell_type": "code", "execution_count": 4, "id": "3e39ebf3-6940-40f1-a7f8-90efb3846f7b", "metadata": {}, "outputs": [], "source": [ "resp = client.create_kx_cluster(\n", " environmentId=ENV_ID, \n", " clusterName=CLUSTER_NAME,\n", " clusterType='RDB',\n", " releaseLabel = '1.0',\n", " databases=DATABASE_CONFIG,\n", " capacityConfiguration=CAPACITY_CONFIG,\n", " savedownStorageConfiguration=SAVEDOWN_CONFIG,\n", " clusterDescription=\"Created with create_cluster_RDB notebook\",\n", " code=CODE_CONFIG,\n", " initializationScript=INIT_SCRIPT,\n", " azMode=AZ_MODE,\n", " availabilityZoneId=AZ_ID,\n", " commandLineArguments=CMD_ARGS,\n", " vpcConfiguration={ \n", " 'vpcId': VPC_ID,\n", " 'securityGroupIds': SECURITY_GROUPS,\n", " 'subnetIds': SUBNET_IDS,\n", " 'ipAddressType': 'IP_V4' }\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "e9946083-c9bb-4ecc-afbd-8c20e284ddf5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cluster: RDB_welcomedb status is PENDING, total wait 0:00:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:00:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:01:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:01:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:02:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:02:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:03:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:03:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:04:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:04:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:05:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:05:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:06:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:06:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:07:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:07:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:08:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:08:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:09:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:09:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:10:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:10:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:11:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:11:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:12:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:12:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:13:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:13:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:14:00, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is CREATING, total wait 0:14:30, waiting 30 sec ...\n", "Cluster: RDB_welcomedb status is now RUNNING, total wait 0:15:00\n", "\n", "** DONE **\n" ] } ], "source": [ "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=CLUSTER_NAME, show_wait=True)\n", "print()\n", "print(\"** DONE **\")" ] }, { "cell_type": "code", "execution_count": null, "id": "389b275e-f916-4915-aa3c-7436a26d2989", "metadata": {}, "outputs": [], "source": [ "# Give permissions time to propogate after cluster creation....\n", "time.sleep(60)" ] }, { "cell_type": "code", "execution_count": 7, "id": "be61a417-938a-432f-a7ba-a9969971b436", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ ":tcps://vpce-0a299a1f762f351b0-uiaa28cm.vpce-svc-061a7d8630e15dd1a.us-east-1.vpce.amazonaws.com:443:bob:Host=vpce-0a299a1f762f351b0-uiaa28cm.vpce-svc-061a7d8630e15dd1a.us-east-1.vpce.amazonaws.com&Port=5000&User=bob&Action=finspace%3AConnectKxCluster&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEP3%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJHMEUCIQCgBdnCSb7VbDcOBBMHmptBNq4UEXM707K0Vz2%2FyuOBbwIgXHOGKdMrs5Pd29AotL3djT65Aoi2Ft5It3QBSxaAjewq%2BAIIRhAAGgw2MTI4NDEzODM1OTQiDFSliLTQPhEn0U%2FunirVAnLKiKasSNUQdclq2RvDDJyf%2BdNe4fYEHwsyBL%2BNcDsCOkgLrXAk3DXDI%2FrhGpB6Jwsir8p8H2HGqye%2B4vWY8o6x8lGwKLbO60AhoFYe7OZVfS0eUOPr5it2p0hj58amb3jvhLVPCB%2FQejQJaaYtgpDlklQZfA2VpMXtArl%2B9rzN3C8NQiOQokuZidg6%2Fc8Y3DYacZ84D13jkCbgc7yaBjS%2BJeESE%2BBBWuIzBq8eFOYhE%2F0KMBc2A2EGo4pQt2ly6U4Kbs6Y7v97h9Wz6wVBmZTYE2E6nrwuneFG%2B9PMTb7yNYtyQc8S%2F4%2BWZd6cJ89lQgURN5WWv1D8Fb0%2BHffaSh4%2BTpuGyRyiWJSMEJYAfGWfbNSV6jRR%2BXM%2BaV1ZDa6nPzXiTggQCKeIceAE0wqdoO1i4ixD3S%2Fo2feYbhQQwm%2BmV1kSWsYAU7mkArVuoll4wsLp1USsMLnc%2FKMGOr8B2EBfwFLIeU%2B0PDseKiOSTnLqB3QKpLiDs%2BvH4WsgvVDXIVzDgGTKv9UbnFx%2FfD8tDIdyq7zAT4DfsR2n86U%2FRC00IAESTy2txZvq3wPy7kYuyNc6IpVmABeygav8n44kucHnQVBL4QhXwYgMH8QFOsQcz7lLiCSSIG6RaXif7%2FfG3ztnmIjwYvVDCitXpZ7F36tmCJA6pN4blKPfc4fUN6XS9yPGfg4pyiTuMoyaI%2BpIGezFPPaRbUAbhDs%2FjXw%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20230606T130145Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=ASIAY5MBRM2VO5NKCGWV%2F20230606%2Fus-east-1%2Ffinspace-apricot%2Faws4_request&X-Amz-Signature=f4f2bbd75b091cded537c13615184d7638d499835cd169a3c8ac13c51dca058d\n" ] } ], "source": [ "conn_str = get_kx_connection_string(client, environmentId=ENV_ID, clusterName=CLUSTER_NAME, userName=KDB_USERNAME, boto_session=session)\n", "print(conn_str)" ] }, { "cell_type": "code", "execution_count": 8, "id": "1c50c578-05e8-49e7-8deb-1f6b94b10221", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
clusterNamestatusclusterTypecapacityConfigurationcommandLineArgumentsclusterDescriptionlastModifiedTimestampcreatedTimestampdatabaseNamecacheConfigurations
HDB_TAQ_2021_2DRUNNINGHDB{'nodeType': 'kx.s.32xlarge', 'nodeCount': 2}[{'key': 's', 'value': '8'}, {'key': 'dbname', 'value': 'TAQ_2021_2D'}, {'key': 'codebase', 'value': 'taqcode'}]Created with create_cluster_TAQ_2D notebook2023-06-06 12:54:08.985000+00:002023-06-06 12:40:07.108000+00:00TAQ_2021_2D[{'cacheType': 'CACHE_1000', 'dbPaths': ['/2021.01.04/', '/2021.01.05/']}]
HDB_basictickdb_20230601RUNNINGHDB{'nodeType': 'kx.s.xlarge', 'nodeCount': 2}[{'key': 's', 'value': '8'}, {'key': 'dbname', 'value': 'basictickdb'}, {'key': 'codebase', 'value': 'basictick'}]Created with create_HDB for basic_tick notebook2023-06-06 11:52:52.544000+00:002023-06-06 11:35:14.234000+00:00basictickdb[{'cacheType': 'CACHE_1000', 'dbPaths': ['/']}]
HDB_welcomedbRUNNINGHDB{'nodeType': 'kx.s.2xlarge', 'nodeCount': 3}[{'key': 's', 'value': '4'}, {'key': 'dbname', 'value': 'welcomedb'}, {'key': 'codebase', 'value': 'code'}]Created with create_cluster_HDB notebook2023-06-06 11:50:28.453000+00:002023-06-06 11:32:13.700000+00:00welcomedb[{'cacheType': 'CACHE_1000', 'dbPaths': ['/']}]
RDB_basictickdb_20230601RUNNINGRDB{'nodeType': 'kx.s.2xlarge', 'nodeCount': 1}[{'key': 's', 'value': '8'}, {'key': 'dbname', 'value': 'basictickdb'}, {'key': 'codebase', 'value': 'basictick'}, {'key': 'tphostfile', 'value': 'tickerplant2'}]Created with create_RDB notebook2023-06-06 11:54:07.399000+00:002023-06-06 11:38:47.020000+00:00basictickdb[]
RDB_welcomedbRUNNINGRDB{'nodeType': 'kx.s.2xlarge', 'nodeCount': 1}[{'key': 's', 'value': '8'}, {'key': 'dbname', 'value': 'welcomedb'}, {'key': 'codebase', 'value': 'code'}]Created with create_cluster_RDB notebook2023-06-06 11:47:08.448000+00:002023-06-06 11:32:06.677000+00:00welcomedb[]
cluster_create_delete_db_20230606_1132RUNNINGHDB{'nodeType': 'kx.s.xlarge', 'nodeCount': 2}[{'key': 's', 'value': '4'}, {'key': 'dbname', 'value': 'create_delete_db_20230606_1132'}, {'key': 'codebase', 'value': 'code'}]Demo Cluster for database create_delete_db_20230606_11322023-06-06 11:50:54.875000+00:002023-06-06 11:33:18.188000+00:00create_delete_db_20230606_1132[{'cacheType': 'CACHE_1000', 'dbPaths': ['/']}]
hdb-cluster-welcomedbRUNNINGHDB{'nodeType': 'kx.s.xlarge', 'nodeCount': 2}[{'key': 's', 'value': '4'}, {'key': 'dbname', 'value': 'welcomedb'}, {'key': 'codebase', 'value': 'code'}]None2023-06-06 11:44:19.914000+00:002023-06-06 11:26:51.517000+00:00welcomedb[{'cacheType': 'CACHE_1000', 'dbPaths': ['/']}]
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "cdf = get_clusters(client, ENV_ID)\n", "\n", "display(cdf)" ] }, { "cell_type": "code", "execution_count": 9, "id": "86f33240-bb12-49f3-8d9c-5783c25eb182", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Last Run: 2023-06-06 13:01:48.118482\n" ] } ], "source": [ "print( f\"Last Run: {datetime.datetime.now()}\" )" ] }, { "cell_type": "code", "execution_count": null, "id": "e4f0d8e3-7a79-4f5d-b68b-c7e5b44c6685", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" } }, "nbformat": 4, "nbformat_minor": 5 }