ManagedkdbInsights/processing_data/delete_all.ipynb (293 lines of code) (raw):
{
"cells": [
{
"cell_type": "markdown",
"id": "28bea13b-67bd-4a0e-8eab-3b8ffd37259e",
"metadata": {},
"source": [
"# CSV Processing: Delete Everything\n",
"This notebook will tear down all resources created by the create_all notebook."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "0d5f1d4a-ed45-44e3-bf75-9bdb75fcddbb",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import subprocess\n",
"import boto3\n",
"import json\n",
"import datetime\n",
"\n",
"import pykx as kx\n",
"\n",
"from managed_kx import *\n",
"from env import *\n",
"from config import *\n",
"\n",
"# Selective Deletion or All\n",
"DELETE_ALL=True\n",
"#DELETE_ALL=False\n",
"\n",
"DELETE_CLUSTERS=True\n",
"DELETE_SCALING_GROUP=False\n",
"DELETE_DATAVIEW=True\n",
"DELETE_VOLUME=False\n",
"DELETE_DATABASE=True\n",
"\n",
"if DELETE_DATABASE:\n",
" DELETE_DATAVIEW=True\n",
"\n",
"if DELETE_VOLUME:\n",
" DELETE_DATAVIEW=True\n",
"\n",
"if DELETE_ALL:\n",
" DELETE_CLUSTERS=True\n",
" DELETE_SCALING_GROUP=True\n",
" DELETE_DATAVIEW=True\n",
" DELETE_VOLUME=True\n",
" DELETE_DATABASE=True\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3cfe7d89-9f5d-4ceb-ac8c-1f5054a6f15a",
"metadata": {},
"outputs": [],
"source": [
"# Using credentials and create service client\n",
"session = boto3.Session()\n",
"\n",
"# create finspace client\n",
"client = session.client(service_name='finspace')"
]
},
{
"cell_type": "markdown",
"id": "eeaeb9f8-d4ce-4a8a-b367-4a86a9dae6a6",
"metadata": {},
"source": [
"## Delete all resources\n",
"Using service APIs to delete all resources, if resource does not exist, assume it was already deleted and continue.\n",
"\n",
"### APIs Used\n",
"[get_kx_cluster](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_cluster.html) \n",
"[delete_kx_cluster](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_cluster.html) \n",
"[get_kx_dataview](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_dataview.html) \n",
"[delete_kx_dataview](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_dataview.html) \n",
"[get_kx_volume](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_volume.html) \n",
"[delete_kx_volume](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_volume.html) \n",
"[get_kx_scaling_group](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_scaling_group.html) \n",
"[delete_kx_scaling_group](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_scaling_group.html) \n",
"[get_kx_database](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/get_kx_database.html) \n",
"[delete_kx_database](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/finspace/client/delete_kx_database.html) "
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "dd0b0fc2-83ca-402a-bb33-09834a9cbe76",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Deleting Clusters...\n",
"\tDelete: demo_csv_cluster\n",
"\tDeleted: demo_csv_cluster\n",
"\tDelete: demo_hdb_cluster\n",
"\tDeleted: demo_hdb_cluster\n",
"** Deleted 2 Clusters **\n",
"\n",
"Deleting Dataview: DEMO_DB_VIEW...\n",
"delete: DEMO_DB_VIEW\n",
"Dataview: DEMO_DB_VIEW status is DELETING, total wait 0:00:00, waiting 30 sec ...\n",
"Dataview: DEMO_DB_VIEW status is DELETING, total wait 0:00:30, waiting 30 sec ...\n",
"Dataview: DEMO_DB_VIEW status is DELETING, total wait 0:01:00, waiting 30 sec ...\n",
"Dataview: DEMO_DB_VIEW status is DELETING, total wait 0:01:30, waiting 30 sec ...\n",
"Dataview: DEMO_DB_VIEW not found\n",
"** Deleted View **\n",
"\n",
"Deleting Database: DEMO_DB...\n",
"delete: DEMO_DB\n",
"** Deleted Database **\n",
"\n",
"Done Deleting\n"
]
}
],
"source": [
"# Clusters\n",
"#-------------------------------------------------------------------------------------------\n",
"delete_status = ['RUNNING', 'CREATING', 'ACTIVE', 'CREATE_FAILED']\n",
"deleting_status = ['DELETING']\n",
"\n",
"if DELETE_CLUSTERS:\n",
" print(\"Deleting Clusters...\")\n",
"\n",
" # Clusters\n",
" for c in clusters:\n",
" this_c = get_kx_cluster(client=client, clusterName=c)\n",
" if this_c is None:\n",
" print(f\"Not Found: {c}\")\n",
" continue\n",
" elif this_c['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {c} Status: {this_c['status']}\")\n",
" continue\n",
"\n",
" print(f\"\\tDelete: {c}\")\n",
" client.delete_kx_cluster(environmentId=ENV_ID, clusterName=c)\n",
" print(f\"\\tDeleted: {c}\")\n",
"\n",
" print(f\"** Deleted {len(clusters)} Clusters **\")\n",
"\n",
"# Dataview\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_DATAVIEW:\n",
" print()\n",
" print(f\"Deleting Dataview: {DBVIEW_NAME}...\")\n",
" this_view = get_kx_dataview(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME)\n",
"\n",
" if this_view is None:\n",
" print(f\"Not Found: {DBVIEW_NAME}\")\n",
" elif this_view['status'] in deleting_status:\n",
" wait_for_dataview_status(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted View **\")\n",
" elif this_view['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {DBVIEW_NAME} Status: {this_view['status']}\")\n",
" else:\n",
" print(f\"delete: {DBVIEW_NAME}\")\n",
" client.delete_kx_dataview(environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME)\n",
" wait_for_dataview_status(client=client, environmentId=ENV_ID, databaseName=DB_NAME, dataviewName=DBVIEW_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted View **\")\n",
"\n",
"# Volume\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_VOLUME:\n",
" print()\n",
" print(f\"Deleting Volume: {VOLUME_NAME}...\")\n",
" this_volume = get_kx_volume(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME)\n",
"\n",
" if this_volume is None:\n",
" print(f\"Not Found: {VOLUME_NAME}\")\n",
" elif this_volume['status'] in deleting_status:\n",
" wait_for_volume_status(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Volume **\")\n",
" elif this_volume['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {VOLUME_NAME} Status: {this_volume['status']}\")\n",
" else:\n",
" print(f\"Delete: {VOLUME_NAME}\")\n",
" client.delete_kx_volume(environmentId=ENV_ID, volumeName=VOLUME_NAME)\n",
" wait_for_volume_status(client=client, environmentId=ENV_ID, volumeName=VOLUME_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Volume **\")\n",
"\n",
"# Scaling Group\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_SCALING_GROUP:\n",
" print()\n",
" print(f\"Deleting Scaling Group: {SCALING_GROUP_NAME}...\")\n",
" this_sg = get_kx_scaling_group(client=client, environmentId=ENV_ID, scalingGroupName=SCALING_GROUP_NAME)\n",
"\n",
" if this_sg is None:\n",
" print(f\"Not Found: {SCALING_GROUP_NAME}\")\n",
" elif this_sg['status'] in deleting_status:\n",
" wait_for_scaling_group_status(client=client, environmentId=ENV_ID, scalingGroupName = SCALING_GROUP_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Scaling Group **\")\n",
" elif this_sg['status'] not in delete_status:\n",
" print(f\"Cannot Delete: {SCALING_GROUP_NAME} Status: {this_sg['status']}\")\n",
" else:\n",
" print(f\"Delete: {SCALING_GROUP_NAME}\")\n",
" client.delete_kx_scaling_group(environmentId=ENV_ID, scalingGroupName=SCALING_GROUP_NAME)\n",
" wait_for_scaling_group_status(client=client, environmentId=ENV_ID, scalingGroupName = SCALING_GROUP_NAME, status='DELETED', show_wait=True)\n",
" print(\"** Deleted Scaling Group **\")\n",
"\n",
"# Database\n",
"#-------------------------------------------------------------------------------------------\n",
"if DELETE_DATABASE:\n",
" print()\n",
" print(f\"Deleting Database: {DB_NAME}...\")\n",
" this_db = get_kx_database(client=client, environmentId=ENV_ID, databaseName=DB_NAME)\n",
"\n",
" if this_db is None:\n",
" print(f\"Not Found: {DB_NAME}\")\n",
" else:\n",
" print(f\"delete: {DB_NAME}\")\n",
" client.delete_kx_database(environmentId=ENV_ID, databaseName=DB_NAME) \n",
" print(\"** Deleted Database **\")\n",
"\n",
"print(\"\\nDone Deleting\")"
]
},
{
"cell_type": "markdown",
"id": "75955733-2255-4490-9514-0b2d6dfab055",
"metadata": {},
"source": [
"# Delete any directories created\n",
"[create_all.ipynb](create_all.ipynb) created an empty database in demo directory and extracted market data tarball to algoseek-marketdata."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b18dc2fa-268d-44f5-99c9-325baad47846",
"metadata": {},
"outputs": [],
"source": [
"!rm -rf demo algoseek-marketdata"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "86f33240-bb12-49f3-8d9c-5783c25eb182",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Last Run: 2024-11-26 18:05:54.409222\n"
]
}
],
"source": [
"print( f\"Last Run: {datetime.datetime.now()}\" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7413c1b0-1862-46e1-b485-cd4ae861154a",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "conda_pytorch_p310",
"language": "python",
"name": "conda_pytorch_p310"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}