ManagedkdbInsights/torq/refresh_code.ipynb (868 lines of code) (raw):
{
"cells": [
{
"cell_type": "markdown",
"id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e",
"metadata": {},
"source": [
"# TorQ: Create Everything\n",
"This notebook will use the AWS boto3 APIs to create the needed resources for a TorQ based application.\n",
"\n",
"## AWS Resources Created\n",
"- Database \n",
"- Changeset to add data to database \n",
"- Scaling Group that will contain all clusters \n",
"- Shared Volume \n",
"- Dataview of database on the shared volume \n",
"- Clusters\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import subprocess\n",
"import boto3\n",
"import json\n",
"import datetime\n",
"\n",
"import pykx as kx\n",
"\n",
"from managed_kx import *\n",
"\n",
"from env import *\n",
"from config import *\n",
"\n",
"# ----------------------------------------------------------------\n",
"\n",
"TORQ_CODEBASE=\"TorQ\"\n",
"TORQ_FINSPACE_CODEBASE=\"TorQ-Amazon-FinSpace-Starter-Pack\"\n",
"\n",
"# Source data directory\n",
"SOURCE_DATA_DIR=f\"{TORQ_FINSPACE_CODEBASE}/hdb\"\n",
"\n",
"# Code directory\n",
"CODEBASE=\"torq_app\"\n",
"\n",
"# unique code zipfile name\n",
"today=datetime.datetime.now().strftime(\"%Y%m%d_%H%M\") \n",
"CODE_ZIP=f\"{CODEBASE}_{today}.zip\"\n",
"\n",
"# S3 Destinations\n",
"S3_CODE_PATH=\"code\"\n",
"S3_DATA_PATH=\"data\"\n",
"\n",
"NODE_TYPE=\"kx.sg.4xlarge\"\n",
"\n",
"DATABASE_CONFIG=[{ \n",
" 'databaseName': DB_NAME,\n",
" 'dataviewName': DBVIEW_NAME\n",
" }]\n",
"CODE_CONFIG={ 's3Bucket': S3_BUCKET, 's3Key': f'{S3_CODE_PATH}/{CODE_ZIP}' }\n",
"\n",
"NAS1_CONFIG= {\n",
" 'type': 'SSD_250',\n",
" 'size': 1200\n",
"}\n",
"\n",
"#update_type=\"NO_RESTART\"\n",
"#update_type=\"ROLLING\"\n",
"update_type=\"FORCE\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a",
"metadata": {},
"outputs": [],
"source": [
"# Using credentials and create service client\n",
"session = boto3.Session()\n",
"\n",
"# create finspace client\n",
"client = session.client(service_name='finspace')"
]
},
{
"cell_type": "markdown",
"id": "0f0c06ab-4dcb-4cc6-abc9-2c77ff3a4242",
"metadata": {},
"source": [
"# Stage Fresh Code to S3\n",
"Code to be used in this application must be staged to an S3 bucket the service can read from, that code will then be deployed to the clusters as part of their creation workflow."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "b502a0a5-8610-4fc8-b6b7-04c47e89ba75",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"upload: ./torq_app_20241203_2103.zip to s3://kdb-demo-829845998889-kms/code/torq_app_20241203_2103.zip\n",
"2024-12-03 21:03:48 8925181 torq_app_20241203_2103.zip\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# zip the code\n",
"os.system(f\"zip -q -r {CODE_ZIP} TorQ/ TorQ-Amazon-FinSpace-Starter-Pack/ -x '*.ipynb_checkpoints*' -x 'TorQ-Amazon-FinSpace-Starter-Pack/hdb/*' -x '*.git*' -x 'TorQ/tests/*' -x 'TorQ-Amazon-FinSpace-Starter-Pack/terraform-deployment/*'\")\n",
"\n",
"cp = \"\"\n",
"\n",
"# copy code to S3\n",
"if AWS_ACCESS_KEY_ID is not None:\n",
" cp = f\"\"\"\n",
"export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}\n",
"export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}\n",
"export AWS_SESSION_TOKEN={AWS_SESSION_TOKEN}\n",
"\"\"\"\n",
"\n",
"cp += f\"\"\"\n",
"aws s3 cp --exclude .DS_Store {CODE_ZIP} s3://{S3_BUCKET}/code/{CODE_ZIP}\n",
"aws s3 ls s3://{S3_BUCKET}/code/{CODE_ZIP}\n",
"\"\"\"\n",
" \n",
"# execute the S3 copy\n",
"os.system(cp)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d0a75d53-e567-4a2d-b47b-170829d79f63",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: discovery1 status is now RUNNING, total wait 0:00:00\n",
"Updating: discovery1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'discovery1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'discovery'},\n",
" {'key': 'procname', 'value': 'discovery1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '2'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': 'fb78a703-3e7b-40d1-ab30-841760e2fc6d',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:03:50 GMT',\n",
" 'x-amzn-requestid': 'fb78a703-3e7b-40d1-ab30-841760e2fc6d',\n",
" 'x-amz-apigw-id': 'CO7IVHn4IAMEJ_w=',\n",
" 'x-amzn-trace-id': 'Root=1-674f7234-6661b7d404b6f1bd7f3484b0',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': 'J2MA2ikfXkWCL2GDcJrMN0cHK-ei4dyAOaSQ0ensypzKVif6KoISTw=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: rdb1 status is now RUNNING, total wait 0:00:00\n",
"Updating: rdb1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'rdb1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'rdb'},\n",
" {'key': 'procname', 'value': 'rdb1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '2'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': '60c74198-f157-4850-beaa-2dee9f730a59',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:03:51 GMT',\n",
" 'x-amzn-requestid': '60c74198-f157-4850-beaa-2dee9f730a59',\n",
" 'x-amz-apigw-id': 'CO7IpHDmIAMEp7w=',\n",
" 'x-amzn-trace-id': 'Root=1-674f7236-0848f5be398a87a6693efafa',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': 'pbsS0CSFSP3DKYrlRuhnYZC4zfFaVdNKXyPgO_J5VUZPmouyxmaObw=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: hdb1 status is now RUNNING, total wait 0:00:00\n",
"Updating: hdb1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'hdb1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'hdb'},\n",
" {'key': 'procname', 'value': 'hdb1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '4'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': '33348f23-6ff9-4d70-9299-5e66397780de',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:03:51 GMT',\n",
" 'x-amzn-requestid': '33348f23-6ff9-4d70-9299-5e66397780de',\n",
" 'x-amz-apigw-id': 'CO7IvEJMoAMEBow=',\n",
" 'x-amzn-trace-id': 'Root=1-674f7237-25ea16f06dbf4d6413b8ef15',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': '9jE1U_YiskoJGoEBAL1penMx1sPlpFjLpq1eW4OzTmuC5M968l9hGA=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: discovery1 status is UPDATING, total wait 0:00:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:00:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:01:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:01:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:02:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:02:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:03:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:03:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:04:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:04:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:05:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:05:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:06:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:06:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:07:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:07:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:08:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:08:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:09:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:09:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:10:00, waiting 30 sec ...\n",
"Cluster: discovery1 status is UPDATING, total wait 0:10:30, waiting 30 sec ...\n",
"Cluster: discovery1 status is now RUNNING, total wait 0:11:00\n",
"Cluster: rdb1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: gateway1 status is now RUNNING, total wait 0:00:00\n",
"Updating: gateway1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'gateway1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'gateway'},\n",
" {'key': 'procname', 'value': 'gateway1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '2'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': '04a5bdf4-1de9-4816-b116-4655eca04236',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:14:58 GMT',\n",
" 'x-amzn-requestid': '04a5bdf4-1de9-4816-b116-4655eca04236',\n",
" 'x-amz-apigw-id': 'CO8wpHoCIAMEQ_Q=',\n",
" 'x-amzn-trace-id': 'Root=1-674f74d0-5edebd9670e0cdd80f4b0744',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': 'sYaGuzG5fa7O6PnKLSZHGzb-R5BM-12NeGeZcuKhLlS-v8zLlzm9YA=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: feed1 status is now RUNNING, total wait 0:00:00\n",
"Updating: feed1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'feed1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'tradeFeed'},\n",
" {'key': 'procname', 'value': 'tradeFeed1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '2'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': '71e012e8-1881-495b-8b1b-076f1e73c082',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:14:58 GMT',\n",
" 'x-amzn-requestid': '71e012e8-1881-495b-8b1b-076f1e73c082',\n",
" 'x-amz-apigw-id': 'CO8w6GT8IAMEikg=',\n",
" 'x-amzn-trace-id': 'Root=1-674f74d2-13da01ed3c554caa7bf6d728',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': 'De1408uwUtrp_tSlKt1-gQu2ZlkFckEp7ZVGRKxotuLZ3Oaq5YsRnA=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: monitor1 status is now RUNNING, total wait 0:00:00\n",
"Updating: monitor1\n"
]
},
{
"data": {
"text/plain": [
"{'environmentId': 'jlcenjvtkgzrdek2qqv7ic',\n",
" 'clusterName': 'monitor1',\n",
" 'code': {'s3Bucket': 'kdb-demo-829845998889-kms',\n",
" 's3Key': 'code/torq_app_20241203_2103.zip'},\n",
" 'deploymentConfiguration': {'deploymentStrategy': 'FORCE'},\n",
" 'initializationScript': 'TorQ-Amazon-FinSpace-Starter-Pack/finspace_torq.q',\n",
" 'commandLineArguments': [{'key': 'proctype', 'value': 'monitor'},\n",
" {'key': 'procname', 'value': 'monitor1'},\n",
" {'key': 'jsonlogs', 'value': 'true'},\n",
" {'key': 'noredirect', 'value': 'true'},\n",
" {'key': 's', 'value': '1'}]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'ResponseMetadata': {'RequestId': '527ecf5d-e028-42a5-a606-954f7e6d102b',\n",
" 'HTTPStatusCode': 200,\n",
" 'HTTPHeaders': {'content-type': 'application/json',\n",
" 'content-length': '2',\n",
" 'connection': 'keep-alive',\n",
" 'date': 'Tue, 03 Dec 2024 21:14:59 GMT',\n",
" 'x-amzn-requestid': '527ecf5d-e028-42a5-a606-954f7e6d102b',\n",
" 'x-amz-apigw-id': 'CO8xAGsyIAMEXjQ=',\n",
" 'x-amzn-trace-id': 'Root=1-674f74d2-64d640840a5b6ee05c322c88',\n",
" 'x-cache': 'Miss from cloudfront',\n",
" 'via': '1.1 e37edb91bbf6b4d78a917647aaf7d0d0.cloudfront.net (CloudFront)',\n",
" 'x-amz-cf-pop': 'IAD55-P1',\n",
" 'x-amz-cf-id': 'uvbeCLO6b8FJkK7vrcCrBlTpZAfQ5IdjTEAe_3Vx_90EX0j4zo8-IA=='},\n",
" 'RetryAttempts': 0}}"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# update clusters in same order as they where created\n",
"# this will account for dependencies\n",
"\n",
"for c in clusters:\n",
" # wait for a cluster?\n",
" if c['type'] == \"WAIT\":\n",
" wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=c['name'], show_wait=True)\n",
" continue\n",
"\n",
" # cluster must be running to update it\n",
" wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=c['name'], show_wait=True)\n",
"\n",
" cluster_name = c['name']\n",
" cluster_type = c['type']\n",
" cluster_init = c['init']\n",
" cluster_args = c['args']\n",
"\n",
" print(f\"Updating: {cluster_name}\")\n",
"\n",
" update_args = {}\n",
"\n",
" update_args['environmentId']=ENV_ID\n",
" update_args['clusterName']=cluster_name \n",
" update_args['code']= CODE_CONFIG\n",
" update_args['deploymentConfiguration']={\n",
" 'deploymentStrategy': update_type\n",
" }\n",
"\n",
" if update_type != \"NO_RESTART\":\n",
" update_args['initializationScript']=cluster_init\n",
" update_args['commandLineArguments']=cluster_args\n",
"\n",
" display( update_args )\n",
"\n",
" resp = client.update_kx_cluster_code_configuration( **update_args )\n",
"\n",
" display(resp)"
]
},
{
"cell_type": "markdown",
"id": "e8491391-8c01-4190-a2bd-23fa888bf781",
"metadata": {},
"source": [
"## Wait for all clusters to finish updating"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "0946ca26-c4b0-410d-ade5-18a47bf2318a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cluster: discovery1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: rdb1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: hdb1 status is UPDATING, total wait 0:00:00, waiting 30 sec ...\n",
"Cluster: hdb1 status is now RUNNING, total wait 0:00:30\n",
"Cluster: discovery1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: rdb1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: gateway1 status is UPDATING, total wait 0:00:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:00:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:01:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:01:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:02:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:02:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:03:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:03:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:04:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:04:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:05:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:05:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:06:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:06:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:07:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:07:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:08:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:08:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:09:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:09:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:10:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:10:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:11:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:11:30, waiting 30 sec ...\n",
"Cluster: gateway1 status is UPDATING, total wait 0:12:00, waiting 30 sec ...\n",
"Cluster: gateway1 status is now RUNNING, total wait 0:12:30\n",
"Cluster: feed1 status is now RUNNING, total wait 0:00:00\n",
"Cluster: monitor1 status is now RUNNING, total wait 0:00:00\n",
"** ALL DONE **\n"
]
}
],
"source": [
"# Wait for all clusters to start\n",
"for c in clusters:\n",
" cluster_name = c['name']\n",
" wait_for_cluster_status(client, environmentId=ENV_ID, clusterName=cluster_name, show_wait=True)\n",
"\n",
"print(\"** ALL DONE **\")"
]
},
{
"cell_type": "markdown",
"id": "230207e8-c297-4d7e-af65-4396fa5b4deb",
"metadata": {},
"source": [
"# List Clusters"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "1c50c578-05e8-49e7-8deb-1f6b94b10221",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>clusterName</th>\n",
" <th>status</th>\n",
" <th>clusterType</th>\n",
" <th>capacityConfiguration</th>\n",
" <th>commandLineArguments</th>\n",
" <th>clusterDescription</th>\n",
" <th>lastModifiedTimestamp</th>\n",
" <th>createdTimestamp</th>\n",
" <th>databaseName</th>\n",
" <th>cacheConfigurations</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>discovery1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'discovery1'}, {'key': 'proctype', 'value': 'discovery'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:14:44.294000+00:00</td>\n",
" <td>2024-12-03 20:01:35.746000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>feed1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'tradeFeed1'}, {'key': 'proctype', 'value': 'tradeFeed'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:27:04.539000+00:00</td>\n",
" <td>2024-12-03 20:18:27.837000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>gateway1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'gateway1'}, {'key': 'proctype', 'value': 'gateway'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:27:44.434000+00:00</td>\n",
" <td>2024-12-03 20:18:24.649000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>hdb1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '4'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'hdb1'}, {'key': 'proctype', 'value': 'hdb'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:15:12.760000+00:00</td>\n",
" <td>2024-12-03 20:01:41.735000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>monitor1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '1'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'monitor1'}, {'key': 'proctype', 'value': 'monitor'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:25:26.933000+00:00</td>\n",
" <td>2024-12-03 20:18:30.995000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>rdb1</td>\n",
" <td>RUNNING</td>\n",
" <td>GP</td>\n",
" <td>None</td>\n",
" <td>[{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'rdb1'}, {'key': 'proctype', 'value': 'rdb'}]</td>\n",
" <td>Created with create_all notebook</td>\n",
" <td>2024-12-03 21:14:43.708000+00:00</td>\n",
" <td>2024-12-03 20:01:39.007000+00:00</td>\n",
" <td>finspace-database</td>\n",
" <td>None</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" clusterName status clusterType capacityConfiguration \\\n",
"3 discovery1 RUNNING GP None \n",
"4 feed1 RUNNING GP None \n",
"5 gateway1 RUNNING GP None \n",
"6 hdb1 RUNNING GP None \n",
"7 monitor1 RUNNING GP None \n",
"8 rdb1 RUNNING GP None \n",
"\n",
" commandLineArguments \\\n",
"3 [{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'discovery1'}, {'key': 'proctype', 'value': 'discovery'}] \n",
"4 [{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'tradeFeed1'}, {'key': 'proctype', 'value': 'tradeFeed'}] \n",
"5 [{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'gateway1'}, {'key': 'proctype', 'value': 'gateway'}] \n",
"6 [{'key': 's', 'value': '4'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'hdb1'}, {'key': 'proctype', 'value': 'hdb'}] \n",
"7 [{'key': 's', 'value': '1'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'monitor1'}, {'key': 'proctype', 'value': 'monitor'}] \n",
"8 [{'key': 's', 'value': '2'}, {'key': 'jsonlogs', 'value': 'true'}, {'key': 'noredirect', 'value': 'true'}, {'key': 'procname', 'value': 'rdb1'}, {'key': 'proctype', 'value': 'rdb'}] \n",
"\n",
" clusterDescription lastModifiedTimestamp \\\n",
"3 Created with create_all notebook 2024-12-03 21:14:44.294000+00:00 \n",
"4 Created with create_all notebook 2024-12-03 21:27:04.539000+00:00 \n",
"5 Created with create_all notebook 2024-12-03 21:27:44.434000+00:00 \n",
"6 Created with create_all notebook 2024-12-03 21:15:12.760000+00:00 \n",
"7 Created with create_all notebook 2024-12-03 21:25:26.933000+00:00 \n",
"8 Created with create_all notebook 2024-12-03 21:14:43.708000+00:00 \n",
"\n",
" createdTimestamp databaseName cacheConfigurations \n",
"3 2024-12-03 20:01:35.746000+00:00 finspace-database None \n",
"4 2024-12-03 20:18:27.837000+00:00 finspace-database None \n",
"5 2024-12-03 20:18:24.649000+00:00 finspace-database None \n",
"6 2024-12-03 20:01:41.735000+00:00 finspace-database None \n",
"7 2024-12-03 20:18:30.995000+00:00 finspace-database None \n",
"8 2024-12-03 20:01:39.007000+00:00 finspace-database None "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"cdf = get_clusters(client, environmentId=ENV_ID)\n",
"\n",
"all_clusters = []\n",
"\n",
"for c in clusters:\n",
" all_clusters.append(c['name'])\n",
"\n",
"if cdf is not None:\n",
" cdf = cdf[cdf['clusterName'].isin(all_clusters)]\n",
"\n",
"display(cdf)"
]
},
{
"cell_type": "markdown",
"id": "b31bce7d-637c-4ace-b94c-58d816c8b31f",
"metadata": {},
"source": [
"# Logs\n",
"You can use AWS contols and log trail to monitor logs, below are terminal based ways to tail the log as well for each cluster."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "120ada93-a4d9-406c-9e8d-0b5b0d146437",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy/Paste any line below in a terminal with AWS CLI to tail the logs of that cluster\n",
"\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/discovery1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/rdb1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/hdb1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/discovery1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/rdb1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/gateway1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/feed1 --follow --format json\n",
"aws logs tail /aws/vendedlogs/finspace/jlcenjvtkgzrdek2qqv7ic/monitor1 --follow --format json\n"
]
}
],
"source": [
"def get_log_group(environmentId: str, clusterName: str):\n",
" return f\"/aws/vendedlogs/finspace/{environmentId}/{clusterName}\"\n",
"\n",
"#log_format=\"detailed\"\n",
"#log_format=\"short\"\n",
"log_format=\"json\"\n",
"\n",
"print(\"Copy/Paste any line below in a terminal with AWS CLI to tail the logs of that cluster\")\n",
"print()\n",
"for c in clusters:\n",
" cluster_name = c['name']\n",
"\n",
" lg = get_log_group(environmentId=ENV_ID, clusterName=cluster_name)\n",
"\n",
" print( f\"aws logs tail {lg} --follow --format {log_format}\" )"
]
},
{
"cell_type": "markdown",
"id": "9e91a23b-b100-4763-9c50-c819f5824202",
"metadata": {},
"source": [
"# All Processes Running"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "86f33240-bb12-49f3-8d9c-5783c25eb182",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Last Run: 2024-12-03 21:28:07.699778\n"
]
}
],
"source": [
"print( f\"Last Run: {datetime.datetime.now()}\" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "81f05fa3-fbe0-443b-986e-428fb1ca4ae6",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "conda_python3",
"language": "python",
"name": "conda_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}