ManagedkdbInsights/boto/create

{ "cells": [ { "cell_type": "markdown", "id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e", "metadata": {}, "source": [ "# Create Cluster: HDB\n", "This notebook will create (start) an HDB cluster on a named database." ] }, { "cell_type": "code", "execution_count": null, "id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb", "metadata": {}, "outputs": [], "source": [ "import os\n", "import boto3\n", "import json\n", "import datetime\n", "\n", "import pykx as kx\n", "\n", "from managed_kx import *\n", "from env import *\n", "\n", "# Managed KX Database and Cluster names to create\n", "DB_NAME=\"welcomedb\"\n", "\n", "SEC_THREADS='4'\n", "CLUSTER_NAME=f\"HDB_{DB_NAME}\"\n", "\n", "# Cluster Settings\n", "CODEBASE=\"code\"\n", "S3_CODE_PATH=\"code\"\n", "\n", "DB_PATHS = [ '/' ]\n", "\n", "CAPACITY_CONFIG={ 'nodeCount': 3, 'nodeType': 'kx.s.xlarge'}\n", "DATABASE_CONFIG=[{ 'databaseName': DB_NAME,'cacheConfigurations': [{'dbPaths': DB_PATHS, 'cacheType': 'CACHE_1000' }] }]\n", "CACHE_CONFIG=[{'type': 'CACHE_1000', 'size':1200}]\n", "\n", "CODE_CONFIG={ 's3Bucket': S3_BUCKET, 's3Key': f'{S3_CODE_PATH}/{CODEBASE}.zip' }\n", "\n", "INIT_SCRIPT=f'init.q'\n", "CMD_ARGS=[\n", " { 'key': 's', 'value': SEC_THREADS }, \n", " { 'key': 'dbname', 'value': DB_NAME}, \n", "]\n", "\n", "# VPC Configuration\n", "VPC_CONFIG={ \n", " 'vpcId': VPC_ID,\n", " 'securityGroupIds': SECURITY_GROUPS,\n", " 'subnetIds': SUBNET_IDS,\n", " 'ipAddressType': 'IP_V4' \n", "}\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a", "metadata": {}, "outputs": [], "source": [ "# Using credentials and create service client\n", "session = boto3.Session()\n", "\n", "# create finspace client\n", "client = session.client(service_name='finspace')" ] }, { "cell_type": "markdown", "id": "1cd63f1e-0f36-410d-ab75-95fc2031d221", "metadata": {}, "source": [ "## Check Database" ] }, { "cell_type": "code", "execution_count": null, "id": "f9266a73-d208-4c6a-a50e-9c5b77e99704", "metadata": {}, "outputs": [], "source": [ "note_str = \"\"\n", "\n", "c_set_list = []\n", "\n", "try:\n", " c_set_list = client.list_kx_changesets(environmentId=ENV_ID, databaseName=DB_NAME)['kxChangesets']\n", "except:\n", " note_str = \"<<Could not get changesets>>\"\n", "\n", "print(100*\"=\")\n", "print(f\"Database: {DB_NAME}, Changesets: {len(c_set_list)} {note_str}\")\n", "print(100*\"=\")\n", "\n", "# sort by create time\n", "c_set_list = sorted(c_set_list, key=lambda d: d['createdTimestamp']) \n", "\n", "for c in c_set_list:\n", " c_set_id = c['changesetId']\n", " print(f\"Changeset ({c['status']}): {c_set_id}: Created: {c['createdTimestamp']}\")\n", " c_rqs = client.get_kx_changeset(environmentId=ENV_ID, databaseName=DB_NAME, changesetId=c_set_id)['changeRequests']\n", "\n", " chs_pdf = pd.DataFrame.from_dict(c_rqs).style.hide(axis='index')\n", " display(chs_pdf)" ] }, { "cell_type": "markdown", "id": "67476efe-d308-4158-9e24-8fbe71509f76", "metadata": {}, "source": [ "## Create Cluster" ] }, { "cell_type": "code", "execution_count": null, "id": "a581026e-6104-41a8-8c18-d76e06ed027d", "metadata": {}, "outputs": [], "source": [ "# zip the code\n", "#os.system(f\"zip -r -X {CODEBASE}.zip {CODEBASE} -x '*.ipynb_checkpoints*'\")\n", "os.system(f\"cd {CODEBASE}; zip -r -X ../{CODEBASE}.zip . -x '*.ipynb_checkpoints*';\")\n", "\n", "# copy code to S3\n", "cp = \"\"\n", "\n", "if AWS_ACCESS_KEY_ID is not None:\n", " cp = f\"\"\"\n", "export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}\n", "export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}\n", "export AWS_SESSION_TOKEN={AWS_SESSION_TOKEN}\n", "\"\"\"\n", "\n", "cp += f\"\"\"\n", "aws s3 cp --exclude .DS_Store {CODEBASE}.zip s3://{S3_BUCKET}/code/{CODEBASE}.zip\n", "aws s3 ls s3://{S3_BUCKET}/code/\n", "\"\"\"\n", " \n", "# execute the S3 copy\n", "os.system(cp)" ] }, { "cell_type": "code", "execution_count": null, "id": "3e39ebf3-6940-40f1-a7f8-90efb3846f7b", "metadata": {}, "outputs": [], "source": [ "resp = client.create_kx_cluster(\n", " environmentId=ENV_ID, \n", " clusterName=CLUSTER_NAME,\n", " clusterType='HDB',\n", " releaseLabel = '1.0',\n", " capacityConfiguration=CAPACITY_CONFIG,\n", " databases=DATABASE_CONFIG,\n", " cacheStorageConfigurations=CACHE_CONFIG,\n", " clusterDescription=\"Created with create_cluster_HDB notebook\",\n", " code=CODE_CONFIG,\n", " initializationScript=INIT_SCRIPT,\n", " commandLineArguments=CMD_ARGS,\n", " azMode=AZ_MODE,\n", " availabilityZoneId=AZ_ID,\n", " vpcConfiguration=VPC_CONFIG\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "e9946083-c9bb-4ecc-afbd-8c20e284ddf5", "metadata": {}, "outputs": [], "source": [ "wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=CLUSTER_NAME, show_wait=True)\n", "print()\n", "print(\"** DONE **\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d32ea7cd-06a2-4972-87e9-3645cf7e1c45", "metadata": { "tags": [] }, "outputs": [], "source": [ "# get the connection string to the cluster\n", "conn_str = get_kx_connection_string(client, \n", " environmentId=ENV_ID, clusterName=CLUSTER_NAME, \n", " userName=KDB_USERNAME, boto_session=session)\n", "\n", "# parse the connection string to components\n", "host, port, username, password = parse_connection_string(conn_str)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e6544dc3-93aa-41dd-931d-38d52fd9efdb", "metadata": { "tags": [] }, "outputs": [], "source": [ "%%q --host $host --port $port --user $username --pass $password\n", "tables[]" ] }, { "cell_type": "code", "execution_count": null, "id": "1c50c578-05e8-49e7-8deb-1f6b94b10221", "metadata": {}, "outputs": [], "source": [ "cdf = get_clusters(client, environmentId=ENV_ID)\n", "\n", "display(cdf)" ] }, { "cell_type": "code", "execution_count": null, "id": "86f33240-bb12-49f3-8d9c-5783c25eb182", "metadata": {}, "outputs": [], "source": [ "print( f\"Last Run: {datetime.datetime.now()}\" )" ] }, { "cell_type": "code", "execution_count": null, "id": "e4f0d8e3-7a79-4f5d-b68b-c7e5b44c6685", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }

ManagedkdbInsights/boto/create_cluster.ipynb (274 lines of code) (raw):