ManagedkdbInsights/basic_tick_V3/delete_all.ipynb (355 lines of code) (raw):
{
"cells": [
{
"cell_type": "markdown",
"id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e",
"metadata": {},
"source": [
"# BasicTick: Delete Everything\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import subprocess\n",
"import boto3\n",
"import json\n",
"import datetime\n",
"\n",
"import pykx as kx\n",
"\n",
"from managed_kx import *\n",
"from env import *\n",
"\n",
"# Cluster names and database\n",
"from config import *\n",
"\n",
"# Selective Deletion or All\n",
"DELETE_ALL=True\n",
"#DELETE_ALL=False\n",
"\n",
"DELETE_CLUSTERS=True\n",
"DELETE_SCALING_GROUP=False\n",
"DELETE_DATAVIEW=False\n",
"DELETE_VOLUME=False\n",
"DELETE_DATABASE=False\n",
"\n",
"if DELETE_DATABASE:\n",
" DELETE_DATAVIEW=True\n",
"\n",
"if DELETE_VOLUME:\n",
" DELETE_DATAVIEW=True\n",
"\n",
"if DELETE_ALL:\n",
" DELETE_CLUSTERS=True\n",
" DELETE_SCALING_GROUP=True\n",
" DELETE_DATAVIEW=True\n",
" DELETE_VOLUME=True\n",
" DELETE_DATABASE=True\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a",
"metadata": {},
"outputs": [],
"source": [
"# Using credentials and create service client\n",
"session = boto3.Session()\n",
"\n",
"# create finspace client\n",
"client = session.client(service_name='finspace')"
]
},
{
"cell_type": "markdown",
"id": "eeaeb9f8-d4ce-4a8a-b367-4a86a9dae6a6",
"metadata": {},
"source": [
"# Delete All Resources\n",
"\n",
"Using service APIs to delete all resources, if resource does not exist, assume it was already deleted and continue.\n",
"\n",
"## APIs Used\n",
"[get_kx_cluster](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_cluster.html) \n",
"[delete_kx_cluster](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_cluster.html) \n",
"[get_kx_dataview](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_dataview.html) \n",
"[delete_kx_dataview](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_dataview.html) \n",
"[get_kx_volume](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_volume.html) \n",
"[delete_kx_volume](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_volume.html) \n",
"[get_kx_scaling_group](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_scaling_group.html) \n",
"[delete_kx_scaling_group](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_scaling_group.html) \n",
"[get_kx_database](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_database.html) \n",
"[delete_kx_database](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_database.html) \n",
"\n",
"## Kill FeedHandler and Others\n",
"You can connect to the running (not finspace cluster) q instance and have it exit.\n",
"\n",
"### Example\n",
"```\n",
"$ q\n",
"q) h:hopen`::5030\n",
"q) @[h; \"exit 0\", {}]\n",
"```\n",
"\n",
"Here we do the equivalent from Python"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "dd0b0fc2-83ca-402a-bb33-09834a9cbe76",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Stopping local FeedHandler ...\n",
"Feedhandler killed\n"
]
}
],
"source": [
"print(\"Stopping local FeedHandler ...\")\n",
"\n",
"ret=os.system('pkill -f \"python feedhandler_pykx.py\"')\n",
"\n",
"if ret == 0:\n",
" print(\"Feedhandler killed\")\n",
"else:\n",
" print(\"Feedhandler was not running\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b2967b5a-2f64-4462-beef-d50b70d4e6a8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Deleting Clusters...\n",
"\tDelete: TP_basictickdb\n",
"\tDeleted: TP_basictickdb\n",
"\tDelete: RDB_basictickdb\n",
"\tDeleted: RDB_basictickdb\n",
"\tDelete: HDB_basictickdb\n",
"\tDeleted: HDB_basictickdb\n",
"\tDelete: GATEWAY_basictickdb\n",
"\tDeleted: GATEWAY_basictickdb\n",
"\tDelete: RTS_basictickdb\n",
"\tDeleted: RTS_basictickdb\n",
"** Deleted 5 Clusters **\n",
"\n",
"Deleting Dataview: basictickdb_DBVIEW...\n",
"delete: basictickdb_DBVIEW\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:00:00, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:00:30, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:01:00, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:01:30, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:02:00, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:02:30, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:03:00, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:03:30, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW status is DELETING, total wait 0:04:00, waiting 30 sec ...\n",
"Dataview: basictickdb_DBVIEW not found\n",
"** Deleted View **\n",
"\n",
"Deleting Volume: RDB_TP_SHARED...\n",
"Delete: RDB_TP_SHARED\n",
"Volume: RDB_TP_SHARED not found\n",
"** Deleted Volume **\n",
"\n",
"Deleting Database: basictickdb...\n",
"Delete: RDB_TP_SHARED\n",
"** Deleted Database **\n",
"\n",
"Deleting Scaling Group: SCALING_GROUP_basictickdb...\n",
"Delete: SCALING_GROUP_basictickdb\n",
"Scaling Group: SCALING_GROUP_basictickdb not found\n",
"** Deleted Scaling Group **\n",
"\n",
"Done Deleting\n"
]
}
],
"source": [
"# Clusters\n",
"#-------------------------------------------------------------------------------------------\n",
"delete_status = ['RUNNING', 'CREATING', 'ACTIVE', 'CREATE_FAILED']\n",
"deleting_status = ['DELETING']\n",
"\n",
"if DELETE_CLUSTERS:\n",
" print(\"Deleting Clusters...\")\n",
"\n",
" for k,v in all_clusters.items():\n",
" print(f\"\\tDelete: {v}\")\n",
" try:\n",
" client.delete_kx_cluster(environmentId=ENV_ID, clusterName=v)\n",
" print(f\"\\tDeleted: {v}\")\n",
" except:\n",
" print(f\"\\tAlready Deleted? {v}\")\n",
" next\n",
"\n",
" print(f\"** Deleted {len(all_clusters)} Clusters **\")\n",
"\n",
"# Dataview\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_DATAVIEW:\n",
" print()\n",
" print(f\"Deleting Dataview: {DBVIEW_NAME}...\")\n",
" this_view = get_kx_dataview(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME)\n",
"\n",
" if this_view is None:\n",
" print(f\"Not Found: {DBVIEW_NAME}\")\n",
" elif this_view['status'] in deleting_status:\n",
" wait_for_dataview_status(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted View **\")\n",
" elif this_view['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {DBVIEW_NAME} Status: {this_view['status']}\")\n",
" else:\n",
" print(f\"delete: {DBVIEW_NAME}\")\n",
" client.delete_kx_dataview(environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME)\n",
" wait_for_dataview_status(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted View **\")\n",
"\n",
"# Volume\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_VOLUME:\n",
" print()\n",
" print(f\"Deleting Volume: {VOLUME_NAME}...\")\n",
" this_volume = get_kx_volume(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME)\n",
"\n",
" if this_volume is None:\n",
" print(f\"Not Found: {VOLUME_NAME}\")\n",
" elif this_volume['status'] in deleting_status:\n",
" wait_for_volume_status(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Volume **\")\n",
" elif this_volume['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {VOLUME_NAME} Status: {this_volume['status']}\")\n",
" else:\n",
" print(f\"Delete: {VOLUME_NAME}\")\n",
" client.delete_kx_volume(environmentId=ENV_ID, volumeName=VOLUME_NAME)\n",
" wait_for_volume_status(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Volume **\")\n",
"\n",
"# Volume\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_DATABASE:\n",
" print()\n",
" print(f\"Deleting Database: {DB_NAME}...\")\n",
" this_database = get_kx_database(client=client, environmentId=ENV_ID, databaseName=DB_NAME)\n",
"\n",
" if this_database is None:\n",
" print(f\"Not Found: {DB_NAME}\")\n",
" else:\n",
" print(f\"Delete: {VOLUME_NAME}\")\n",
" client.delete_kx_database(environmentId=ENV_ID, databaseName=DB_NAME)\n",
" print(\"** Deleted Database **\")\n",
"\n",
"# Scaling Group\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_SCALING_GROUP:\n",
" print()\n",
" print(f\"Deleting Scaling Group: {SCALING_GROUP_NAME}...\")\n",
" this_sg = get_kx_scaling_group(client=client, environmentId=ENV_ID, scalingGroupName=SCALING_GROUP_NAME)\n",
"\n",
" if this_sg is None:\n",
" print(f\"Not Found: {SCALING_GROUP_NAME}\")\n",
" elif this_sg['status'] in deleting_status:\n",
" wait_for_scaling_group_status(client=client, environmentId=ENV_ID, scalingGroupName = SCALING_GROUP_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Scaling Group **\")\n",
" elif this_sg['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {SCALING_GROUP_NAME} Status: {this_sg['status']}\")\n",
" else:\n",
" print(f\"Delete: {SCALING_GROUP_NAME}\")\n",
" client.delete_kx_scaling_group(environmentId=ENV_ID, scalingGroupName=SCALING_GROUP_NAME)\n",
" wait_for_scaling_group_status(client=client, environmentId=ENV_ID, scalingGroupName = SCALING_GROUP_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Scaling Group **\")\n",
"\n",
"print(\"\\nDone Deleting\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "869d94f7-b860-4023-90d5-bfa0af5dafec",
"metadata": {},
"outputs": [],
"source": [
"!pkill -f \"python feedhandler_pykx.py\"\n",
"\n",
"# Kills the feedhandler"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "dcc43fcd-4302-42de-85f2-30558d82f500",
"metadata": {},
"outputs": [],
"source": [
"!rm -rf hdb\n",
"# removes the hdb directory"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "86f33240-bb12-49f3-8d9c-5783c25eb182",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Last Run: 2024-11-26 17:36:22.966602\n"
]
}
],
"source": [
"print( f\"Last Run: {datetime.datetime.now()}\" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e4f0d8e3-7a79-4f5d-b68b-c7e5b44c6685",
"metadata": {
"tags": []
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "conda_python3",
"language": "python",
"name": "conda_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}