{ "cells": [ { "cell_type": "markdown", "id": "e13084db-c69f-4e70-aa79-acfd2159093a", "metadata": {}, "source": [ "# List Databases" ] }, { "cell_type": "code", "execution_count": 1, "id": "3698017d-42ed-4280-afa1-723f19ae398c", "metadata": { "tags": [] }, "outputs": [], "source": [ "import os\n", "import boto3\n", "import json\n", "import datetime\n", "\n", "from managed_kx import *\n", "from env_2 import *" ] }, { "cell_type": "code", "execution_count": 2, "id": "ed241efe-523e-4d2f-8e1b-94a09adf42e7", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using variables ...\n" ] } ], "source": [ "# triggers credential get\n", "session=None\n", "\n", "try:\n", " subprocess.call([\"which\", \"ada\"])\n", " os.system(f\"ada credentials update --account={ACCOUNT_ID} --provider=isengard --role=Admin --once\")\n", "except: \n", " None\n", "\n", "if AWS_ACCESS_KEY_ID is None:\n", " print(\"Using Defaults ...\")\n", " # create AWS session: using access variables\n", " session = boto3.Session()\n", "else:\n", " print(\"Using variables ...\")\n", " session = boto3.Session(\n", " aws_access_key_id=AWS_ACCESS_KEY_ID,\n", " aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n", " aws_session_token=AWS_SESSION_TOKEN\n", " )\n", "\n", "# create finspace client\n", "client = session.client(service_name='finspace', endpoint_url=ENDPOINT_URL)" ] }, { "cell_type": "markdown", "id": "6b7294c7-d9a3-4fb2-b671-12ead3e17e83", "metadata": {}, "source": [ "# Databases" ] }, { "cell_type": "code", "execution_count": 3, "id": "4d600940-ffbf-41a2-8df0-a7a7fc0b594b", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
databaseNamecreatedTimestamplastModifiedTimestamp
TAQ_2021H12023-05-30 22:55:27.556000+00:002023-05-30 22:59:43.785000+00:00
TAQ_2021_2D2023-05-30 22:54:17.321000+00:002023-05-30 22:54:17.945000+00:00
create_delete_db2023-05-30 22:09:40.541000+00:002023-05-30 22:47:19.805000+00:00
welcomedb2023-05-30 22:53:12.481000+00:002023-05-30 22:53:13.165000+00:00
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "db_list = list_kx_databases(client, environmentId=ENV_ID)\n", "db_list = sorted(db_list, key=lambda d: d['databaseName']) \n", "\n", "db_pdf = pd.DataFrame.from_dict(db_list).style.hide(axis='index')\n", "display(db_pdf)" ] }, { "cell_type": "markdown", "id": "57a48a33-6426-42c1-933e-7cc92990e3e2", "metadata": {}, "source": [ "# Databases and Changesets " ] }, { "cell_type": "code", "execution_count": 4, "id": "7078c851-337b-44b9-8942-9dc867771972", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "====================================================================================================\n", "Database: TAQ_2021H1, Changesets: \n", "TAQ kdb database\n", "Bytes: 1,635,996,702,039 Changesets: 5 Files: 993\n", "----------------------------------------------------------------------------------------------------\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
changesetIdcreatedTimestampactiveFromTimestamplastModifiedTimestampstatus
0wsQ3ccETqSvyo9ldBuoyYw2023-05-30 22:59:43.785000+00:002023-05-30 23:00:25.586000+00:002023-05-30 22:59:43.785000+00:00COMPLETED
1GMQ3cVlvvAwrhUzCYFb2BA2023-05-30 22:58:50.724000+00:002023-05-30 22:59:33.829000+00:002023-05-30 22:58:50.724000+00:00COMPLETED
25sQ3cMgxGMuyNyuiApXwsA2023-05-30 22:57:36.358000+00:002023-05-30 22:58:43.211000+00:002023-05-30 22:57:36.358000+00:00COMPLETED
3mMQ3cDd7XJcOecX6Sq9XJw2023-05-30 22:56:22.267000+00:002023-05-30 22:57:23.423000+00:002023-05-30 22:56:22.267000+00:00COMPLETED
49MQ3b89NSnY6pXQhLVIkNA2023-05-30 22:55:28.927000+00:002023-05-30 22:56:18.094000+00:002023-05-30 22:55:28.927000+00:00COMPLETED
\n", "
" ], "text/plain": [ " changesetId createdTimestamp \\\n", "0 wsQ3ccETqSvyo9ldBuoyYw 2023-05-30 22:59:43.785000+00:00 \n", "1 GMQ3cVlvvAwrhUzCYFb2BA 2023-05-30 22:58:50.724000+00:00 \n", "2 5sQ3cMgxGMuyNyuiApXwsA 2023-05-30 22:57:36.358000+00:00 \n", "3 mMQ3cDd7XJcOecX6Sq9XJw 2023-05-30 22:56:22.267000+00:00 \n", "4 9MQ3b89NSnY6pXQhLVIkNA 2023-05-30 22:55:28.927000+00:00 \n", "\n", " activeFromTimestamp lastModifiedTimestamp status \n", "0 2023-05-30 23:00:25.586000+00:00 2023-05-30 22:59:43.785000+00:00 COMPLETED \n", "1 2023-05-30 22:59:33.829000+00:00 2023-05-30 22:58:50.724000+00:00 COMPLETED \n", "2 2023-05-30 22:58:43.211000+00:00 2023-05-30 22:57:36.358000+00:00 COMPLETED \n", "3 2023-05-30 22:57:23.423000+00:00 2023-05-30 22:56:22.267000+00:00 COMPLETED \n", "4 2023-05-30 22:56:18.094000+00:00 2023-05-30 22:55:28.927000+00:00 COMPLETED " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "====================================================================================================\n", "Database: TAQ_2021_2D, Changesets: \n", "TAQ kdb database\n", "Bytes: 23,554,923,456 Changesets: 1 Files: 17\n", "----------------------------------------------------------------------------------------------------\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
changesetIdcreatedTimestampactiveFromTimestamplastModifiedTimestampstatus
0xMQ3b0Sr31twz2fuQCjxdQ2023-05-30 22:54:17.944000+00:002023-05-30 22:55:06.006000+00:002023-05-30 22:54:17.944000+00:00COMPLETED
\n", "
" ], "text/plain": [ " changesetId createdTimestamp \\\n", "0 xMQ3b0Sr31twz2fuQCjxdQ 2023-05-30 22:54:17.944000+00:00 \n", "\n", " activeFromTimestamp lastModifiedTimestamp status \n", "0 2023-05-30 22:55:06.006000+00:00 2023-05-30 22:54:17.944000+00:00 COMPLETED " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "====================================================================================================\n", "Database: create_delete_db, Changesets: \n", "Welcome kdb database\n", "Bytes: 34,526,532 Changesets: 3 Files: 41\n", "----------------------------------------------------------------------------------------------------\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
changesetIdcreatedTimestampactiveFromTimestamplastModifiedTimestampstatus
0vsQ3bBP9NEmjohJ0F2hW5g2023-05-30 22:47:19.805000+00:002023-05-30 22:47:50.346000+00:002023-05-30 22:47:19.805000+00:00COMPLETED
18sQ3XgXSa87YOIKc5sIgLw2023-05-30 22:16:42.534000+00:002023-05-30 22:16:50.670000+00:002023-05-30 22:16:42.534000+00:00COMPLETED
2nsQ3XGgWIomChGVdUkzS6A2023-05-30 22:13:05.710000+00:002023-05-30 22:13:27.077000+00:002023-05-30 22:13:05.710000+00:00COMPLETED
3zMQ3Wtj34piBnRT4SoeDsw2023-05-30 22:09:41.361000+00:00NaT2023-05-30 22:09:41.361000+00:00FAILED
\n", "
" ], "text/plain": [ " changesetId createdTimestamp \\\n", "0 vsQ3bBP9NEmjohJ0F2hW5g 2023-05-30 22:47:19.805000+00:00 \n", "1 8sQ3XgXSa87YOIKc5sIgLw 2023-05-30 22:16:42.534000+00:00 \n", "2 nsQ3XGgWIomChGVdUkzS6A 2023-05-30 22:13:05.710000+00:00 \n", "3 zMQ3Wtj34piBnRT4SoeDsw 2023-05-30 22:09:41.361000+00:00 \n", "\n", " activeFromTimestamp lastModifiedTimestamp status \n", "0 2023-05-30 22:47:50.346000+00:00 2023-05-30 22:47:19.805000+00:00 COMPLETED \n", "1 2023-05-30 22:16:50.670000+00:00 2023-05-30 22:16:42.534000+00:00 COMPLETED \n", "2 2023-05-30 22:13:27.077000+00:00 2023-05-30 22:13:05.710000+00:00 COMPLETED \n", "3 NaT 2023-05-30 22:09:41.361000+00:00 FAILED " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "====================================================================================================\n", "Database: welcomedb, Changesets: \n", "Welcome kdb database\n", "Bytes: 34,526,532 Changesets: 1 Files: 41\n", "----------------------------------------------------------------------------------------------------\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
changesetIdcreatedTimestampactiveFromTimestamplastModifiedTimestampstatus
0VsQ3bsYloWVaDQSZp7fWmw2023-05-30 22:53:13.165000+00:002023-05-30 22:53:20.534000+00:002023-05-30 22:53:13.165000+00:00COMPLETED
\n", "
" ], "text/plain": [ " changesetId createdTimestamp \\\n", "0 VsQ3bsYloWVaDQSZp7fWmw 2023-05-30 22:53:13.165000+00:00 \n", "\n", " activeFromTimestamp lastModifiedTimestamp status \n", "0 2023-05-30 22:53:20.534000+00:00 2023-05-30 22:53:13.165000+00:00 COMPLETED " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Last Run: 2023-05-30 23:15:12.584851\n" ] } ], "source": [ "for d in db_list:\n", " db_name = d['databaseName']\n", " dump_database(client, ENV_ID, db_name)\n", " \n", "print( f\"Last Run: {datetime.datetime.now()}\" )" ] }, { "cell_type": "code", "execution_count": null, "id": "7ab440c1-99c8-45b2-b465-2baa9958f2cf", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" } }, "nbformat": 4, "nbformat_minor": 5 }