{ "cells": [ { "cell_type": "markdown", "id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e", "metadata": {}, "source": [ "# BasicTick: Create Cluster: GW\n", "This notebook will create (start) a Gateway cluster." ] }, { "cell_type": "code", "execution_count": 1, "id": "83c6572a-0972-4867-8c02-ea2c6c98427a", "metadata": { "jupyter": { "source_hidden": true }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%html\n", "" ] }, { "cell_type": "markdown", "id": "f17a02c0-4f56-455c-a28a-dd102a88201c", "metadata": { "tags": [] }, "source": [ "## Setup\n", "\n", "### Node Types\n", "|Type|Mem (GB)|vCPUs|\n", "|:---|---:|---:|\n", "|kx.s.large|12|2|\n", "|kx.s.xlarge|27|4|\n", "|kx.s.2xlarge|54|8|\n", "|kx.s.4xlarge|108|16|\n", "|kx.s.8xlarge|216|32|\n", "|kx.s.16xlarge|432|64|\n", "|kx.s.32xlarge|864|128|\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb", "metadata": {}, "outputs": [], "source": [ "import os\n", "import boto3\n", "import json\n", "import datetime\n", "\n", "from managed_kx import *\n", "from env_kdb_1 import *\n", "\n", "# Cluster names and database\n", "from basictick_setup import *\n", "\n", "# Managed KX Database and Cluster names to create\n", "SEC_THREADS='4'\n", "\n", "# Cluster Settings\n", "CODEBASE=\"basictick\"\n", "S3_CODE_PATH=\"code\"\n", "\n", "CAPACITY_CONFIG={ 'nodeCount': 1, 'nodeType': 'kx.s.2xlarge'}\n", "CODE_CONFIG={ 's3Bucket': S3_BUCKET, 's3Key': f'{S3_CODE_PATH}/{CODEBASE}.zip' }\n", "INIT_SCRIPT=f'{CODEBASE}/gwmkdbcluster.q'\n", "\n", "CMD_ARGS=[\n", " { 'key': 's', 'value': SEC_THREADS }, \n", " { 'key': 'codebase', 'value': CODEBASE}, \n", " { 'key': 'rdb_name', 'value': RDB_CLUSTER_NAME}, \n", " { 'key': 'hdb_name', 'value': HDB_CLUSTER_NAME}, \n", "]" ] }, { "cell_type": "code", "execution_count": 3, "id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using variables ...\n" ] } ], "source": [ "# triggers credential get\n", "session=None\n", "\n", "try:\n", " # aws: use ada for credentials\n", " subprocess.call([\"which\", \"ada\"])\n", " os.system(f\"ada credentials update --account={ACCOUNT_ID} --provider=isengard --role=Admin --once\")\n", "except: \n", " None\n", "\n", "if AWS_ACCESS_KEY_ID is None:\n", " print(\"Using Defaults ...\")\n", " # create AWS session: using access variables\n", " session = boto3.Session()\n", "else:\n", " print(\"Using variables ...\")\n", " session = boto3.Session(\n", " aws_access_key_id=AWS_ACCESS_KEY_ID,\n", " aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n", " aws_session_token=AWS_SESSION_TOKEN\n", " )\n", "\n", "# create finspace client\n", "client = session.client(service_name='finspace', endpoint_url=ENDPOINT_URL)" ] }, { "cell_type": "markdown", "id": "0eee25f1-cdf7-4310-a869-ef4f2a53cef8", "metadata": {}, "source": [ "# Deploy code" ] }, { "cell_type": "code", "execution_count": 4, "id": "b502a0a5-8610-4fc8-b6b7-04c47e89ba75", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "updating: basictick/ (stored 0%)\n", "updating: basictick/connectmkdb.q (deflated 63%)\n", "updating: basictick/example.schema.q (deflated 12%)\n", "updating: basictick/feedmkdb.q (deflated 53%)\n", "updating: basictick/funcDownHandle.q (deflated 33%)\n", "updating: basictick/gwmkdbcluster.q (deflated 61%)\n", "updating: basictick/hdbmkdb.q (deflated 52%)\n", "updating: basictick/loadDep.q (deflated 57%)\n", "updating: basictick/query.q (deflated 19%)\n", "updating: basictick/rdbmkdb.q (deflated 57%)\n", "updating: basictick/tickerplant2 (stored 0%)\n", "updating: basictick/tickerplant3 (stored 0%)\n", "updating: basictick/tp.q (deflated 52%)\n", "updating: basictick/.ipynb_checkpoints/ (stored 0%)\n", "updating: basictick/.ipynb_checkpoints/tickerplant3-checkpoint (stored 0%)\n", "updating: basictick/.ipynb_checkpoints/tickerplant-checkpoint (stored 0%)\n", "updating: basictick/tickerplant (stored 0%)\n", "updating: basictick/.ipynb_checkpoints/rdbmkdb-checkpoint.q (deflated 57%)\n", "updating: basictick/.ipynb_checkpoints/gwmkdbcluster-checkpoint.q (deflated 61%)\n", "updating: basictick/.ipynb_checkpoints/hdbmkdb-checkpoint.q (deflated 52%)\n", "upload: ./basictick.zip to s3://kdb-demo-829845998889-kms/code/basictick.zip\n", "2023-06-05 21:25:21 0 \n", "2023-07-25 22:42:49 10695 basictick.zip\n", "2023-06-26 17:29:47 757 code.zip\n", "2023-07-13 21:39:39 652 taqcode.zip\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "if AWS_ACCESS_KEY_ID is not None:\n", " cp = f\"\"\"\n", "export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}\n", "export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}\n", "export AWS_SESSION_TOKEN={AWS_SESSION_TOKEN}\n", "\n", "zip -r -X {CODEBASE}.zip {CODEBASE} \n", "aws s3 cp {CODEBASE}.zip s3://{S3_BUCKET}/{S3_CODE_PATH}/{CODEBASE}.zip\n", "aws s3 ls {S3_BUCKET}/{S3_CODE_PATH}/\n", "\"\"\"\n", "else:\n", " cp = f\"\"\"\n", "zip -r -X {CODEBASE}.zip {CODEBASE} \n", "aws s3 cp {CODEBASE}.zip s3://{S3_BUCKET}/{S3_CODE_PATH}/{CODEBASE}.zip\n", "aws s3 ls {S3_BUCKET}/{S3_CODE_PATH}/\n", "\"\"\"\n", " \n", "# execute the S3 copy\n", "os.system(cp)" ] }, { "cell_type": "markdown", "id": "67476efe-d308-4158-9e24-8fbe71509f76", "metadata": {}, "source": [ "## Create Cluster" ] }, { "cell_type": "code", "execution_count": 5, "id": "f0a65ef6-d5c1-482b-b38f-c64bb6bb4df3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "*** HDB and RDB are running ***\n" ] } ], "source": [ "# be sure the HDB and RDB are running\n", "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=HDB_CLUSTER_NAME)\n", "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=RDB_CLUSTER_NAME)\n", "\n", "print(\"*** HDB and RDB are running ***\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "3e39ebf3-6940-40f1-a7f8-90efb3846f7b", "metadata": {}, "outputs": [], "source": [ "resp = client.create_kx_cluster(\n", " environmentId=ENV_ID, \n", " clusterName=GW_CLUSTER_NAME,\n", " clusterType='GATEWAY',\n", " releaseLabel = '1.0',\n", " capacityConfiguration=CAPACITY_CONFIG,\n", " clusterDescription=\"Created with create_GW notebook\",\n", " executionRole=EXECUTION_ROLE,\n", " code=CODE_CONFIG,\n", " initializationScript=INIT_SCRIPT,\n", " commandLineArguments=CMD_ARGS,\n", " azMode=AZ_MODE,\n", " availabilityZoneId=AZ_ID,\n", " vpcConfiguration={ \n", " 'vpcId': VPC_ID,\n", " 'securityGroupIds': SECURITY_GROUPS,\n", " 'subnetIds': SUBNET_IDS,\n", " 'ipAddressType': 'IP_V4' }\n", ")" ] }, { "cell_type": "code", "execution_count": 7, "id": "f09cb24b-5d6f-455a-9b45-cc9a8733b32c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'ResponseMetadata': {'RequestId': 'a3fc3f60-b81b-46df-8948-b2b281fdd517',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/json',\n", " 'content-length': '1060',\n", " 'connection': 'keep-alive',\n", " 'date': 'Tue, 25 Jul 2023 22:42:54 GMT',\n", " 'x-amzn-requestid': 'a3fc3f60-b81b-46df-8948-b2b281fdd517',\n", " 'x-amz-apigw-id': 'IpFstHukoAMF-BQ=',\n", " 'x-amzn-trace-id': 'Root=1-64c04fea-5d581ed46a18bf5d687439b3',\n", " 'x-cache': 'Miss from cloudfront',\n", " 'via': '1.1 3d3fd40be4e4bfdd1e1bebf86df63a76.cloudfront.net (CloudFront)',\n", " 'x-amz-cf-pop': 'IAD55-P1',\n", " 'x-amz-cf-id': 'xfibqaK-Kvqt5bTK9AtmmjDUIDBEsudtG-LwBgG3KMnIUu4s0FUqVA=='},\n", " 'RetryAttempts': 0},\n", " 'status': 'PENDING',\n", " 'clusterName': 'GATEWAY_basictickdb',\n", " 'clusterType': 'GATEWAY',\n", " 'clusterDescription': 'Created with create_GW notebook',\n", " 'capacityConfiguration': {'nodeType': 'kx.s.2xlarge', 'nodeCount': 1},\n", " 'releaseLabel': '1.0',\n", " 'vpcConfiguration': {'vpcId': 'vpc-0fe2b9c50f3ad382f',\n", " 'securityGroupIds': ['sg-0c99f1cfb9c3c7fd9'],\n", " 'subnetIds': ['subnet-04052219ec25b062b'],\n", " 'ipAddressType': 'IP_V4'},\n", " 'initializationScript': 'basictick/gwmkdbcluster.q',\n", " 'commandLineArguments': [{'key': 's', 'value': '4'},\n", " {'key': 'codebase', 'value': 'basictick'},\n", " {'key': 'rdb_name', 'value': 'RDB_basictickdb'},\n", " {'key': 'hdb_name', 'value': 'HDB_basictickdb'}],\n", " 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n", " 's3Key': 'code/basictick.zip'},\n", " 'executionRole': 'arn:aws:iam::829845998889:role/kdb-all-user',\n", " 'lastModifiedTimestamp': datetime.datetime(2023, 7, 25, 22, 42, 54, 90000, tzinfo=tzlocal()),\n", " 'azMode': 'SINGLE',\n", " 'availabilityZoneId': 'use1-az6',\n", " 'createdTimestamp': datetime.datetime(2023, 7, 25, 22, 42, 54, 70000, tzinfo=tzlocal())}" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "resp" ] }, { "cell_type": "code", "execution_count": 8, "id": "e9946083-c9bb-4ecc-afbd-8c20e284ddf5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "** DONE **\n" ] } ], "source": [ "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=GW_CLUSTER_NAME)\n", "print(\"** DONE **\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "1c50c578-05e8-49e7-8deb-1f6b94b10221", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
clusterNamestatusclusterTypecapacityConfigurationcommandLineArgumentsclusterDescriptionlastModifiedTimestampcreatedTimestampdatabaseName
0GATEWAY_basictickdbRUNNINGGATEWAY{'nodeType': 'kx.s.2xlarge', 'nodeCount': 1}[{'key': 's', 'value': '4'}, {'key': 'codebase', 'value': 'basictick'}, {'key': 'rdb_name', 'value': 'RDB_basictickdb'}, {'key': 'hdb_name', 'value': 'HDB_basictickdb'}]Created with create_GW notebook2023-07-25 22:55:43.727000+00:002023-07-25 22:42:54.070000+00:00None
\n", "
" ], "text/plain": [ " clusterName status clusterType \\\n", "0 GATEWAY_basictickdb RUNNING GATEWAY \n", "\n", " capacityConfiguration \\\n", "0 {'nodeType': 'kx.s.2xlarge', 'nodeCount': 1} \n", "\n", " commandLineArguments \\\n", "0 [{'key': 's', 'value': '4'}, {'key': 'codebase', 'value': 'basictick'}, {'key': 'rdb_name', 'value': 'RDB_basictickdb'}, {'key': 'hdb_name', 'value': 'HDB_basictickdb'}] \n", "\n", " clusterDescription lastModifiedTimestamp \\\n", "0 Created with create_GW notebook 2023-07-25 22:55:43.727000+00:00 \n", "\n", " createdTimestamp databaseName \n", "0 2023-07-25 22:42:54.070000+00:00 None " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "cdf = get_clusters(client, environmentId=ENV_ID, clusterType=\"GATEWAY\")\n", "\n", "display(cdf)" ] }, { "cell_type": "code", "execution_count": 10, "id": "86f33240-bb12-49f3-8d9c-5783c25eb182", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Last Run: 2023-07-25 22:56:05.137244\n" ] } ], "source": [ "print( f\"Last Run: {datetime.datetime.now()}\" )" ] }, { "cell_type": "code", "execution_count": null, "id": "e4f0d8e3-7a79-4f5d-b68b-c7e5b44c6685", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" } }, "nbformat": 4, "nbformat_minor": 5 }