plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb

{ "cells": [ { "cell_type": "markdown", "id": "f4ab2c33-c072-49e9-93de-da24759113f7", "metadata": {}, "source": [ "# Bootstrap the client with ROOT credentials\n", "Using the python client generated from our OpenAPI spec, we generate a token from our root user's credentials" ] }, { "cell_type": "code", "execution_count": 1, "id": "f982815a-2b48-46ab-96a6-20dad7ec1420", "metadata": {}, "outputs": [], "source": [ "from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI\n", "from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", "from polaris.catalog.api_client import ApiClient as CatalogApiClient\n", "from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", "\n", "polaris_credential = 'root:s3cr3t' # pragma: allowlist secret\n", "\n", "client_id, client_secret = polaris_credential.split(\":\")\n", "client = CatalogApiClient(CatalogApiClientConfiguration(username=client_id,\n", " password=client_secret,\n", " host='http://polaris:8181/api/catalog'))\n", "\n", "oauth_api = IcebergOAuth2API(client)\n", "token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL',\n", " client_id=client_id,\n", " client_secret=client_secret,\n", " grant_type='client_credentials',\n", " _headers={'realm': 'default-realm'})\n" ] }, { "cell_type": "markdown", "id": "4c21f4a1-4129-4dd8-9a6c-fa6eeabfa56e", "metadata": {}, "source": [ "# Create our first catalog\n", "\n", "* Creates a catalog named `polaris_catalog` that writes to a specified location in the Local Filesystem." ] }, { "cell_type": "code", "execution_count": 2, "id": "0f7a311a-9a55-4ff7-a40e-db3c74c53b9b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "PolarisCatalog(type='INTERNAL', name='polaris_demo', properties=CatalogProperties(default_base_location='file:///tmp/polaris/', additional_properties={}), create_timestamp=1745882018864, last_update_timestamp=1745882018864, entity_version=1, storage_config_info=FileStorageConfigInfo(storage_type='FILE', allowed_locations=['file:///tmp', 'file:///tmp/polaris/']))" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from polaris.management import *\n", "\n", "client = ApiClient(Configuration(access_token=token.access_token,\n", " host='http://polaris:8181/api/management/v1'))\n", "root_client = PolarisDefaultApi(client)\n", "\n", "storage_conf = FileStorageConfigInfo(storage_type=\"FILE\", allowed_locations=[\"file:///tmp\"])\n", "catalog_name = 'polaris_demo'\n", "catalog = Catalog(name=catalog_name, type='INTERNAL', properties={\"default-base-location\": \"file:///tmp/polaris/\"},\n", " storage_config_info=storage_conf)\n", "catalog.storage_config_info = storage_conf\n", "root_client.create_catalog(create_catalog_request=CreateCatalogRequest(catalog=catalog))\n", "resp = root_client.get_catalog(catalog_name=catalog.name)\n", "resp" ] }, { "cell_type": "markdown", "id": "6521039f-c25d-4baa-96ae-a4408c0fced0", "metadata": {}, "source": [ "# Utility Functions" ] }, { "cell_type": "code", "execution_count": 3, "id": "e3e42c12-4e01-4577-bdf5-90c2704a5de8", "metadata": {}, "outputs": [], "source": [ "# Creates a principal with the given name\n", "def create_principal(api, principal_name):\n", " principal = Principal(name=principal_name, type=\"SERVICE\")\n", " try:\n", " principal_result = api.create_principal(CreatePrincipalRequest(principal=principal))\n", " return principal_result\n", " except ApiException as e:\n", " if e.status == 409:\n", " return api.rotate_credentials(principal_name=principal_name)\n", " else:\n", " raise e\n", "\n", "# Create a catalog role with the given name\n", "def create_catalog_role(api, catalog, role_name):\n", " catalog_role = CatalogRole(name=role_name)\n", " try:\n", " api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role))\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", " except ApiException as e:\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", " else:\n", " raise e\n", "\n", "# Create a principal role with the given name\n", "def create_principal_role(api, role_name):\n", " principal_role = PrincipalRole(name=role_name)\n", " try:\n", " api.create_principal_role(CreatePrincipalRoleRequest(principal_role=principal_role))\n", " return api.get_principal_role(principal_role_name=role_name)\n", " except ApiException as e:\n", " return api.get_principal_role(principal_role_name=role_name)\n" ] }, { "cell_type": "markdown", "id": "15c250ca-7161-418e-bc52-8bbd88a3e57c", "metadata": {}, "source": [ "# Create a new Principal, Principal Role, and Catalog Role\n", "The new Principal belongs to the `engineer` principal role, which has `CATALOG_MANAGE_CONTENT` privileges on the `polaris_catalog`. \n", "\n", "\n", "`CATALOG_MANAGE_CONTENT` has create/list/read/write privileges on all entities within the catalog. The same privilege could be granted to a namespace, in which case, the engineers could create/list/read/write any entity under that namespace" ] }, { "cell_type": "code", "execution_count": 4, "id": "c5ceb5ca-f977-46c7-b2a6-07dda59e8a8b", "metadata": {}, "outputs": [], "source": [ "# Create the engineer_principal\n", "engineer_principal = create_principal(root_client, \"principle\")\n", "\n", "# Create the principal role\n", "engineer_role = create_principal_role(root_client, \"engineer\")\n", "\n", "# Create the catalog role\n", "manager_catalog_role = create_catalog_role(root_client, catalog, \"manage_catalog\")\n", "\n", "# Grant the catalog role to the principal role\n", "# All principals in the principal role have the catalog role's privileges\n", "root_client.assign_catalog_role_to_principal_role(principal_role_name=engineer_role.name,\n", " catalog_name=catalog.name,\n", " grant_catalog_role_request=GrantCatalogRoleRequest(catalog_role=manager_catalog_role))\n", "\n", "# Assign privileges to the catalog role\n", "# Here, we grant CATALOG_MANAGE_CONTENT\n", "root_client.add_grant_to_catalog_role(catalog.name, manager_catalog_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.CATALOG_MANAGE_CONTENT)))\n", "\n", "# Assign the principal role to the principal\n", "root_client.assign_principal_role(engineer_principal.principal.name, grant_principal_role_request=GrantPrincipalRoleRequest(principal_role=engineer_role))" ] }, { "cell_type": "markdown", "id": "8a04cf15-a327-4ab9-a083-6da4e7dd1623", "metadata": {}, "source": [ "# Create a reader Principal, Principal Role, and Catalog Role\n", "This new principal belongs to the `product_manager` principal role, which is explicitly granted read and list permissions on the catalog.\n", "\n", "Permissions cascade, so permissions granted at the catalog level are inherited by namespaces and tables within the catalog." ] }, { "cell_type": "code", "execution_count": 5, "id": "b51a6433-99c9-46c5-a855-928e30bad6e5", "metadata": {}, "outputs": [], "source": [ "# Create a reader principal\n", "reader_principal = create_principal(root_client, \"mlee\")\n", "\n", "# Create the principal role\n", "pm_role = create_principal_role(root_client, \"product_manager\")\n", "\n", "# Create the catalog role\n", "read_only_role = create_catalog_role(root_client, catalog, \"read_only\")\n", "\n", "# Grant the catalog role to the principal role\n", "root_client.assign_catalog_role_to_principal_role(principal_role_name=pm_role.name,\n", " catalog_name=catalog.name,\n", " grant_catalog_role_request=GrantCatalogRoleRequest(catalog_role=read_only_role))\n", "\n", "# Assign privileges to the catalog role\n", "# Here, the catalog role is granted READ and LIST privileges at the catalog level\n", "# Privileges cascade down\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.TABLE_LIST)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.TABLE_READ_PROPERTIES)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.TABLE_READ_DATA)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.VIEW_LIST)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.VIEW_READ_PROPERTIES)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.NAMESPACE_READ_PROPERTIES)))\n", "root_client.add_grant_to_catalog_role(catalog.name, read_only_role.name,\n", " AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name,\n", " type='catalog',\n", " privilege=CatalogPrivilege.NAMESPACE_LIST)))\n", "\n", "# Assign the principal role to the principal\n", "root_client.assign_principal_role(reader_principal.principal.name, grant_principal_role_request=GrantPrincipalRoleRequest(principal_role=pm_role))" ] }, { "cell_type": "markdown", "id": "14c1e2b3-a0d4-49b5-8e1e-ddb43f98b115", "metadata": {}, "source": [ "# Create a Spark session with the engineer credentials\n", "\n", "* Catalog URI points to our Polaris installation\n", "* Credential set using the client_id and client_secret generated for the principal\n", "* Scope set to `PRINCIPAL_ROLE:ALL`\n", "* `X-Iceberg-Access-Delegation` is set to vended-credentials" ] }, { "cell_type": "code", "execution_count": 6, "id": "fd13f24b-9d59-470d-9be1-660c22dde680", "metadata": { "tags": [] }, "outputs": [], "source": [ "from pyspark.sql import SparkSession\n", "\n", "spark = (SparkSession.builder\n", " .config(\"spark.jars\", \"../polaris_libs/polaris-iceberg-1.8.1-spark-runtime-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT.jar\")\n", " .config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-aws:3.3.4,io.delta:delta-spark_2.12:3.2.1\")\n", " .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n", " .config('spark.sql.iceberg.vectorization.enabled', 'false')\n", "\n", " .config(\"spark.sql.extensions\", \"org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension\")\n", " # Configure the 'polaris' catalog as an Iceberg rest catalog\n", " .config(\"spark.sql.catalog.polaris\", \"org.apache.polaris.spark.SparkCatalog\")\n", " # Specify the rest catalog endpoint \n", " .config(\"spark.sql.catalog.polaris.uri\", \"http://polaris:8181/api/catalog\")\n", " # Enable token refresh\n", " .config(\"spark.sql.catalog.polaris.token-refresh-enabled\", \"true\")\n", " # specify the client_id:client_secret pair\n", " .config(\"spark.sql.catalog.polaris.credential\", f\"{engineer_principal.credentials.client_id}:{engineer_principal.credentials.client_secret}\")\n", "\n", " # Set the warehouse to the name of the catalog we created\n", " .config(\"spark.sql.catalog.polaris.warehouse\", catalog_name)\n", "\n", " # Scope set to PRINCIPAL_ROLE:ALL\n", " .config(\"spark.sql.catalog.polaris.scope\", 'PRINCIPAL_ROLE:ALL')\n", "\n", " # Enable access credential delegation\n", " .config(\"spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation\", 'vended-credentials')\n", "\n", " # AWS configuration\n", " .config(\"spark.hadoop.fs.s3.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\")\n", "\n", " .config(\"spark.sql.catalog.polaris.io-impl\", \"org.apache.iceberg.io.ResolvingFileIO\")\n", " .config(\"spark.sql.catalog.polaris.s3.region\", \"us-west-2\")\n", " .config(\"spark.history.fs.logDirectory\", \"/home/iceberg/spark-events\")).getOrCreate()\n" ] }, { "cell_type": "markdown", "id": "f1cfef99-4a52-433b-ac1d-c92db5f396a3", "metadata": {}, "source": [ "# USE polaris\n", "Tell Spark to use the Polaris catalog" ] }, { "cell_type": "code", "execution_count": 7, "id": "72e9e5fb-b22e-4d38-bb1e-4ca78c0d0f3e", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------+\n", "|namespace|\n", "+---------+\n", "+---------+\n", "\n" ] } ], "source": [ "spark.sql(\"USE polaris\")\n", "spark.sql(\"SHOW NAMESPACES\").show()" ] }, { "cell_type": "markdown", "id": "6b6c5a4a-d469-4364-9249-1a4aeb4d560c", "metadata": {}, "source": [ "# Create Nested Namespaces" ] }, { "cell_type": "code", "execution_count": 8, "id": "54159ab2-5964-49a0-8202-a4b64ee4f9e7", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+\n", "| namespace|\n", "+---------------+\n", "|DELTA_NS.PUBLIC|\n", "+---------------+\n", "\n" ] } ], "source": [ "spark.sql(\"CREATE NAMESPACE IF NOT EXISTS DELTA_NS\")\n", "spark.sql(\"CREATE NAMESPACE IF NOT EXISTS DELTA_NS.PUBLIC\")\n", "spark.sql(\"SHOW NAMESPACES IN DELTA_NS\").show()" ] }, { "cell_type": "markdown", "id": "51a5311e-4a40-4bdc-aaee-b5845e06d020", "metadata": {}, "source": [ "# Create a Delta Table" ] }, { "cell_type": "code", "execution_count": 10, "id": "4abc8426-7f2a-4f3f-9e26-1f1824f870c6", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "DataFrame[]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "spark.sql(\"USE NAMESPACE DELTA_NS.PUBLIC\")\n", "spark.sql(\"\"\"CREATE TABLE IF NOT EXISTS PEOPLE (\n", " id int, name string)\n", "USING delta LOCATION 'file:///tmp/delta_tables/people';\n", "\"\"\")\n", "# You can also use cloud storage like s3. For eample: s3://<bucket-name>/<path-to-table>\n", "# Make the corresponding credentials are set up correctly" ] }, { "cell_type": "code", "execution_count": 11, "id": "13356e64-23ca-4804-a1b9-e9f57f4d14ca", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+---------+-----------+\n", "| namespace|tableName|isTemporary|\n", "+---------------+---------+-----------+\n", "|DELTA_NS.PUBLIC| PEOPLE| false|\n", "+---------------+---------+-----------+\n", "\n" ] } ], "source": [ "spark.sql(\"SHOW TABLES\").show()" ] }, { "cell_type": "markdown", "id": "91fa7c6c-34e0-4bb9-babc-3f3db4778101", "metadata": {}, "source": [ "# It's Empty" ] }, { "cell_type": "code", "execution_count": 12, "id": "ff5a466d-6a67-4f42-a6a6-ac54ec258e54", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---+----+\n", "| id|name|\n", "+---+----+\n", "+---+----+\n", "\n" ] } ], "source": [ "spark.sql(\"SELECT * FROM PEOPLE\").show()" ] }, { "cell_type": "markdown", "id": "f7b297de-ed11-41df-8ed9-e9396c6c4465", "metadata": {}, "source": [ "# Insert some records\n", "Querying again shows some records" ] }, { "cell_type": "code", "execution_count": 13, "id": "d7ab2991-6de9-4105-9f95-4c9f1c18f426", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---+------+\n", "| id| name|\n", "+---+------+\n", "| 3|jonath|\n", "| 1| anna|\n", "| 2| bob|\n", "+---+------+\n", "\n" ] } ], "source": [ "spark.sql(\"INSERT INTO PEOPLE VALUES (1, 'anna'), (2, 'bob'), (3, 'jonath')\")\n", "spark.sql(\"SELECT * FROM PEOPLE\").show()" ] }, { "cell_type": "markdown", "id": "b3d49be8-cf02-4b81-82ce-31e1ee46630a", "metadata": {}, "source": [ "# Create Iceberg Table" ] }, { "cell_type": "code", "execution_count": 14, "id": "8eb5fd0c-20d4-42ce-a823-b6ae43f58313", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DataFrame[]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "spark.sql(\"USE NAMESPACE DELTA_NS.PUBLIC\")\n", "spark.sql(\"\"\"CREATE TABLE IF NOT EXISTS COUNTRY (code string, name string) USING iceberg\"\"\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "54904fd6-96b4-4198-b5b7-2b6a9e6eea1f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+---------+-----------+\n", "| namespace|tableName|isTemporary|\n", "+---------------+---------+-----------+\n", "|DELTA_NS.PUBLIC| COUNTRY| false|\n", "|DELTA_NS.PUBLIC| PEOPLE| false|\n", "+---------------+---------+-----------+\n", "\n" ] } ], "source": [ "spark.sql(\"SHOW TABLES\").show()" ] }, { "cell_type": "markdown", "id": "ae216740-a5cb-4f28-b219-1114d7189546", "metadata": {}, "source": [ "# Insert values for the iceberg table" ] }, { "cell_type": "code", "execution_count": 16, "id": "b1e639be-5a3a-41c6-a782-dd939bc2eea4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----+-------------+\n", "|code| name|\n", "+----+-------------+\n", "| US|United States|\n", "| CA| Canada|\n", "| FR| France|\n", "| IN| India|\n", "+----+-------------+\n", "\n" ] } ], "source": [ "spark.sql(\"INSERT INTO COUNTRY VALUES ('US', 'United States'), ('CA', 'Canada'), ('FR', 'France'), ('IN', 'India')\")\n", "spark.sql(\"SELECT * FROM COUNTRY\").show()" ] }, { "cell_type": "markdown", "id": "2a76a0ba-800b-436c-b617-3725286af58c", "metadata": {}, "source": [ "# Initiate a new Spark session\n", "Change the credentials to the PM's read-only credentials" ] }, { "cell_type": "code", "execution_count": 17, "id": "6f3aac79-bf45-4603-bd64-30eeab4bdfa7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DataFrame[]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# The new spark session inherits everything from the previous session except for the overridden credentials\n", "new_spark = spark.newSession()\n", "new_spark.conf.set(\"spark.sql.catalog.polaris.credential\", f\"{reader_principal.credentials.client_id}:{reader_principal.credentials.client_secret}\")\n", "new_spark.sql(\"USE polaris\")" ] }, { "cell_type": "markdown", "id": "b6ba9acb-2e9d-4ffa-a685-8a85c75f3046", "metadata": {}, "source": [ "# Show Namespace contents\n", "We can still `USE NAMESPACE` and `SHOW TABLES`, which require `READ_NAMESPACE_PROPERTIES` and `LIST_TABLES` privileges respectively" ] }, { "cell_type": "code", "execution_count": 18, "id": "d517424d-8893-4375-ac3b-c532c8682b6a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+---------+-----------+\n", "| namespace|tableName|isTemporary|\n", "+---------------+---------+-----------+\n", "|DELTA_NS.PUBLIC| COUNTRY| false|\n", "|DELTA_NS.PUBLIC| PEOPLE| false|\n", "+---------------+---------+-----------+\n", "\n" ] } ], "source": [ "new_spark.sql(\"USE NAMESPACE DELTA_NS.PUBLIC\")\n", "new_spark.sql(\"SHOW TABLES\").show()" ] }, { "cell_type": "markdown", "id": "ecfba50e-ec5d-41dd-8715-78ea1c1f42e2", "metadata": {}, "source": [ "# Table reads work" ] }, { "cell_type": "code", "execution_count": 19, "id": "7fce4b1f-4d71-4d03-8b60-3e9ca6ca6ddf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----+-------------+\n", "|code| name|\n", "+----+-------------+\n", "| US|United States|\n", "| CA| Canada|\n", "| FR| France|\n", "| IN| India|\n", "+----+-------------+\n", "\n" ] } ], "source": [ "new_spark.sql(\"SELECT * FROM COUNTRY\").show()" ] }, { "cell_type": "markdown", "id": "1f38c9c2-1b9a-4be1-b0ab-42e64c398b78", "metadata": {}, "source": [ "# Drop table fails\n", "NOTE: there is currently no write privilege support for non-iceberg tables" ] }, { "cell_type": "code", "execution_count": 20, "id": "92c57967-bd4e-4ffb-a01d-0b22c334432c", "metadata": {}, "outputs": [], "source": [ "new_spark.sql(\"DROP TABLE COUNTRY\")" ] }, { "cell_type": "markdown", "id": "572bf0ac-10d4-4b46-a2dd-b921b2ea99d4", "metadata": {}, "source": [ "# Drop table using original spark session" ] }, { "cell_type": "code", "execution_count": 21, "id": "ced0c7cc-1ba6-4711-8645-4ce210491e28", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DataFrame[]" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "spark.sql(\"DROP TABLE COUNTRY\")" ] }, { "cell_type": "code", "execution_count": 22, "id": "a32de6ed-96c3-493c-9430-fba91b527929", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+---------+-----------+\n", "| namespace|tableName|isTemporary|\n", "+---------------+---------+-----------+\n", "|DELTA_NS.PUBLIC| PEOPLE| false|\n", "+---------------+---------+-----------+\n", "\n" ] } ], "source": [ "spark.sql(\"SHOW TABLES\").show()" ] }, { "cell_type": "code", "execution_count": 23, "id": "4e3f75a9-c95c-4a74-b12f-9630013e2291", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------------+---------+-----------+\n", "| namespace|tableName|isTemporary|\n", "+---------------+---------+-----------+\n", "|DELTA_NS.PUBLIC| PEOPLE| false|\n", "+---------------+---------+-----------+\n", "\n" ] } ], "source": [ "new_spark.sql(\"SHOW TABLES\").show()" ] }, { "cell_type": "code", "execution_count": 24, "id": "5b78a8e3-a9e7-4c7f-a9af-7a91d4644e1c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DataFrame[]" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "spark.sql(\"DROP TABLE PEOPLE\")" ] }, { "cell_type": "code", "execution_count": 25, "id": "c42563bb-399e-4308-aa23-764192c5005a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------+---------+-----------+\n", "|namespace|tableName|isTemporary|\n", "+---------+---------+-----------+\n", "+---------+---------+-----------+\n", "\n" ] } ], "source": [ "spark.sql(\"SHOW TABLES\").show()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" }, "toc-autonumbering": false, "toc-showmarkdowntxt": false, "toc-showtags": false }, "nbformat": 4, "nbformat_minor": 5 }

plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb (851 lines of code) (raw):