demo-python/code/conference-demos/ignite-2024/azure-ai-search-multimodal-build-demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": { "redirect_from": ["https://github.com/azure/azure-search-vector-samples/blob/main/demo-python/code/e2e-demos/azure-ai-search-multimodal-build-demo.ipynb"] },
   "source": [
    "# Azure AI Search Multimodal Retrieval Demo\n",
    "\n",
    "## Introduction\n",
    "In this demo, we will show you how to create a multimodal (text + images) vector index in Azure AI Search.\n",
    "\n",
    "**This unofficial code sample was created for an Ignite 2024 demo. It's offered \"as-is\" and might not work for all customers and all scenarios.**\n",
    "\n",
    "## Prerequisites\n",
    "- 🐍 Python 3.9 or higher\n",
    "- ☁️ Azure Blob Storage\n",
    "- 🔗 Azure AI Vision Service or Azure AI Multi-Service Account\n",
    "- 🔗 Azure AI Search Service\n",
    "\n",
    "## Features Covered\n",
    "This demo covers the following features:\n",
    "- ✅ Stored=False\n",
    "- ✅ Scalar Quantization to int8\n",
    "- ✅ Reranking w/full precision vectors\n",
    "- ✅ Oversampling\n",
    "- ✅ Integrated Vectorization\n",
    "- ✅ Multi-Vector Search\n",
    "- ✅ Generate Dense Captions with AI Vision Image Analysis API\n",
    "- ✅ **[NEW]** Azure AI Vision Embedding Skill\n",
    "- ✅ **[NEW]** Azure AI Vision Vectorizer\n",
    "- ✅ **[NEW]** Azure AI Vision Latest Multilingual Model\n",
    "- ✅ **[NEW]** Vector Weighting\n",
    "\n",
    "Let's get started!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install azure-search-documents --pre --quiet\n",
    "! pip install openai python-dotenv azure-identity cohere azure-ai-vision-imageanalysis --quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate JSON with text and images via Azure AI Vision Studio\n",
    "https://portal.vision.cognitive.azure.com/demo/image-captioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from uuid import uuid4\n",
    "\n",
    "from azure.ai.vision.imageanalysis import ImageAnalysisClient\n",
    "from azure.ai.vision.imageanalysis.models import VisualFeatures\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.storage.blob import BlobServiceClient\n",
    "\n",
    "# Get environment variables for Azure AI Vision\n",
    "try:\n",
    "    endpoint = os.getenv(\"AZURE_AI_VISION_ENDPOINT\")\n",
    "    key = os.getenv(\"AZURE_AI_VISION_API_KEY\")\n",
    "    connection_string = os.getenv(\"BLOB_CONNECTION_STRING\")\n",
    "    # container_name = os.getenv(\"BLOB_CONTAINER_NAME\")\n",
    "    container_name = \"vector-sandbox\"\n",
    "except KeyError as e:\n",
    "    print(f\"Missing environment variable: {str(e)}\")\n",
    "    print(\"Set them before running this sample.\")\n",
    "    exit()\n",
    "\n",
    "# Create an Image Analysis client\n",
    "client = ImageAnalysisClient(endpoint=endpoint, credential=AzureKeyCredential(key))\n",
    "\n",
    "# Setup for Azure Blob Storage\n",
    "blob_service_client = BlobServiceClient.from_connection_string(connection_string)\n",
    "container_client = blob_service_client.get_container_client(container_name)\n",
    "\n",
    "def get_caption(image_url):\n",
    "    \"\"\"\n",
    "    Get a caption for the image using Azure AI Vision.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        result = client.analyze_from_url(\n",
    "            image_url=image_url,\n",
    "            visual_features=[VisualFeatures.CAPTION, VisualFeatures.READ],\n",
    "            gender_neutral_caption=False\n",
    "        )\n",
    "        if result.caption is not None:\n",
    "            return result.caption.text\n",
    "        else:\n",
    "            return \"No caption available\"\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n",
    "        return \"Error generating caption\"\n",
    "\n",
    "def generate_json_objects():\n",
    "    json_objects = []\n",
    "\n",
    "    # Iterate over the blobs in the container\n",
    "    for blob in container_client.list_blobs():\n",
    "        image_url = f\"https://{blob_service_client.account_name}.blob.core.windows.net/{container_name}/{blob.name}\"\n",
    "        caption = get_caption(image_url)\n",
    "\n",
    "        json_object = {\"id\": str(uuid4()), \"imageUrl\": image_url, \"caption\": caption}\n",
    "        json_objects.append(json_object)\n",
    "\n",
    "    return json_objects\n",
    "\n",
    "def write_to_file(json_objects):\n",
    "    # Write the updated JSON to a file\n",
    "    with open(\"build-demo.json\", \"w\") as json_file:\n",
    "        json.dump(json_objects, json_file, indent=4)\n",
    "\n",
    "json_objects = generate_json_objects()\n",
    "write_to_file(json_objects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient, SearchIndexerClient\n",
    "from azure.search.documents.indexes.models import (\n",
    "    AIServicesVisionParameters,\n",
    "    AIServicesVisionVectorizer,\n",
    "    AIStudioModelCatalogName,\n",
    "    AzureMachineLearningVectorizer,\n",
    "    AzureOpenAIVectorizer,\n",
    "    AzureOpenAIModelName,\n",
    "    AzureOpenAIParameters,\n",
    "    BlobIndexerDataToExtract,\n",
    "    BlobIndexerParsingMode,\n",
    "    CognitiveServicesAccountKey,\n",
    "    DefaultCognitiveServicesAccount,\n",
    "    ExhaustiveKnnAlgorithmConfiguration,\n",
    "    ExhaustiveKnnParameters,\n",
    "    FieldMapping,\n",
    "    HnswAlgorithmConfiguration,\n",
    "    HnswParameters,\n",
    "    IndexerExecutionStatus,\n",
    "    IndexingParameters,\n",
    "    IndexingParametersConfiguration,\n",
    "    InputFieldMappingEntry,\n",
    "    OutputFieldMappingEntry,\n",
    "    ScalarQuantizationCompressionConfiguration,\n",
    "    ScalarQuantizationParameters,\n",
    "    SearchField,\n",
    "    SearchFieldDataType,\n",
    "    SearchIndex,\n",
    "    SearchIndexer,\n",
    "    SearchIndexerDataContainer,\n",
    "    SearchIndexerDataIdentity,\n",
    "    SearchIndexerDataSourceConnection,\n",
    "    SearchIndexerSkillset,\n",
    "    SemanticConfiguration,\n",
    "    SemanticField,\n",
    "    SemanticPrioritizedFields,\n",
    "    SemanticSearch,\n",
    "    SimpleField,\n",
    "    VectorSearch,\n",
    "    VectorSearchAlgorithmKind,\n",
    "    VectorSearchAlgorithmMetric,\n",
    "    VectorSearchProfile,\n",
    "    VisionVectorizeSkill\n",
    ")\n",
    "from azure.search.documents.models import (\n",
    "    HybridCountAndFacetMode,\n",
    "    HybridSearch,\n",
    "    SearchScoreThreshold,\n",
    "    VectorizableTextQuery,\n",
    "    VectorizableImageBinaryQuery,\n",
    "    VectorizableImageUrlQuery,\n",
    "    VectorSimilarityThreshold,\n",
    ")\n",
    "from azure.storage.blob import BlobServiceClient\n",
    "from dotenv import load_dotenv\n",
    "from IPython.display import Image, display, HTML\n",
    "from openai import AzureOpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# Configuration\n",
    "AZURE_AI_VISION_API_KEY = os.getenv(\"AZURE_AI_VISION_API_KEY\")\n",
    "AZURE_AI_VISION_ENDPOINT = os.getenv(\"AZURE_AI_VISION_ENDPOINT\")\n",
    "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "BLOB_CONNECTION_STRING = os.getenv(\"BLOB_CONNECTION_STRING\")\n",
    "BLOB_CONTAINER_NAME = os.getenv(\"BLOB_CONTAINER_NAME\")\n",
    "INDEX_NAME = \"build-multimodal-demo\"\n",
    "SEARCH_SERVICE_API_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using AAD for authentication.\n"
     ]
    }
   ],
   "source": [
    "# User-specified parameter\n",
    "USE_AAD_FOR_SEARCH = True  # Set this to False to use API key for authentication\n",
    "\n",
    "def authenticate_azure_search(api_key=None, use_aad_for_search=False):\n",
    "    if use_aad_for_search:\n",
    "        print(\"Using AAD for authentication.\")\n",
    "        credential = DefaultAzureCredential()\n",
    "    else:\n",
    "        print(\"Using API keys for authentication.\")\n",
    "        if api_key is None:\n",
    "            raise ValueError(\"API key must be provided if not using AAD for authentication.\")\n",
    "        credential = AzureKeyCredential(api_key)\n",
    "    return credential\n",
    "\n",
    "azure_search_credential = authenticate_azure_search(api_key=SEARCH_SERVICE_API_KEY, use_aad_for_search=USE_AAD_FOR_SEARCH)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a blob data source connector on Azure AI Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data source 'build-multimodal-demo-blob' created or updated successfully.\n"
     ]
    }
   ],
   "source": [
    "def create_or_update_data_source(indexer_client, container_name, connection_string, index_name):\n",
    "    \"\"\"\n",
    "    Create or update a data source connection for Azure AI Search.\n",
    "    \"\"\"\n",
    "    container = SearchIndexerDataContainer(name=container_name)\n",
    "    data_source_connection = SearchIndexerDataSourceConnection(\n",
    "        name=f\"{index_name}-blob\",\n",
    "        type=\"azureblob\",\n",
    "        connection_string=connection_string,\n",
    "        container=container\n",
    "    )\n",
    "    try:\n",
    "        indexer_client.create_or_update_data_source_connection(data_source_connection)\n",
    "        print(f\"Data source '{index_name}-blob' created or updated successfully.\")\n",
    "    except Exception as e:\n",
    "        raise Exception(f\"Failed to create or update data source due to error: {e}\")\n",
    "\n",
    "# Create a SearchIndexerClient instance\n",
    "indexer_client = SearchIndexerClient(SEARCH_SERVICE_ENDPOINT, azure_search_credential)\n",
    "\n",
    "# Call the function to create or update the data source\n",
    "create_or_update_data_source(indexer_client, BLOB_CONTAINER_NAME, BLOB_CONNECTION_STRING, INDEX_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a search index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created index: build-multimodal-demo\n"
     ]
    }
   ],
   "source": [
    "def create_fields():\n",
    "    \"\"\"Creates the fields for the search index based on the specified schema.\"\"\"\n",
    "    return [\n",
    "        SimpleField(\n",
    "            name=\"id\", type=SearchFieldDataType.String, key=True, filterable=True\n",
    "        ),\n",
    "        SearchField(name=\"caption\", type=SearchFieldDataType.String, searchable=True),\n",
    "        SearchField(name=\"imageUrl\", type=SearchFieldDataType.String, searchable=True),\n",
    "        SearchField(\n",
    "            name=\"captionVector\",\n",
    "            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
    "            vector_search_dimensions=1024,\n",
    "            vector_search_profile_name=\"myHnswProfile\",\n",
    "            stored=False,\n",
    "        ),\n",
    "        SearchField(\n",
    "            name=\"imageVector\",\n",
    "            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
    "            vector_search_dimensions=1024,\n",
    "            vector_search_profile_name=\"myHnswProfile\",\n",
    "            stored=False,\n",
    "        ),\n",
    "    ]\n",
    "\n",
    "\n",
    "def create_vector_search_configuration():\n",
    "    \"\"\"Creates the vector search configuration.\"\"\"\n",
    "    return VectorSearch(\n",
    "        algorithms=[\n",
    "            HnswAlgorithmConfiguration(\n",
    "                name=\"myHnsw\",\n",
    "                parameters=HnswParameters(\n",
    "                    m=4,\n",
    "                    ef_construction=400,\n",
    "                    ef_search=500,\n",
    "                    metric=VectorSearchAlgorithmMetric.COSINE,\n",
    "                ),\n",
    "            )\n",
    "        ],\n",
    "        compressions=[\n",
    "            ScalarQuantizationCompressionConfiguration(\n",
    "                name=\"myScalarQuantization\",\n",
    "                rerank_with_original_vectors=True,\n",
    "                default_oversampling=10,\n",
    "                parameters=ScalarQuantizationParameters(quantized_data_type=\"int8\"),\n",
    "            )\n",
    "        ],\n",
    "        vectorizers=[\n",
    "            AIServicesVisionVectorizer(\n",
    "                name=\"myAIServicesVectorizer\",\n",
    "                kind=\"aiServicesVision\",\n",
    "                ai_services_vision_parameters=AIServicesVisionParameters(\n",
    "                    model_version=\"2023-04-15\",\n",
    "                    resource_uri=AZURE_AI_VISION_ENDPOINT,\n",
    "                    api_key=AZURE_AI_VISION_API_KEY,\n",
    "                ),\n",
    "            )\n",
    "        ],\n",
    "        profiles=[\n",
    "            VectorSearchProfile(\n",
    "                name=\"myHnswProfile\",\n",
    "                algorithm_configuration_name=\"myHnsw\",\n",
    "                compression_configuration_name=\"myScalarQuantization\",\n",
    "                vectorizer=\"myAIServicesVectorizer\",\n",
    "            )\n",
    "        ],\n",
    "    )\n",
    "\n",
    "\n",
    "def create_search_index(index_client, index_name, fields, vector_search):\n",
    "    \"\"\"Creates or updates a search index.\"\"\"\n",
    "    index = SearchIndex(\n",
    "        name=index_name,\n",
    "        fields=fields,\n",
    "        vector_search=vector_search,\n",
    "    )\n",
    "    index_client.create_or_update_index(index=index)\n",
    "\n",
    "\n",
    "index_client = SearchIndexClient(\n",
    "    endpoint=SEARCH_SERVICE_ENDPOINT, credential=azure_search_credential\n",
    ")\n",
    "fields = create_fields()\n",
    "vector_search = create_vector_search_configuration()\n",
    "\n",
    "# Create the search index with the adjusted schema\n",
    "create_search_index(index_client, INDEX_NAME, fields, vector_search)\n",
    "print(f\"Created index: {INDEX_NAME}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a Skillset    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created skillset: build-multimodal-demo-skillset\n"
     ]
    }
   ],
   "source": [
    "def create_text_embedding_skill():\n",
    "    return VisionVectorizeSkill(\n",
    "        name=\"text-embedding-skill\",\n",
    "        description=\"Skill to generate embeddings for text via Azure AI Vision\",\n",
    "        context=\"/document\",\n",
    "        model_version=\"2023-04-15\",\n",
    "        inputs=[InputFieldMappingEntry(name=\"text\", source=\"/document/caption\")],\n",
    "        outputs=[OutputFieldMappingEntry(name=\"vector\", target_name=\"captionVector\")],\n",
    "    )\n",
    "\n",
    "def create_image_embedding_skill():\n",
    "    return VisionVectorizeSkill(\n",
    "        name=\"image-embedding-skill\",\n",
    "        description=\"Skill to generate embeddings for image via Azure AI Vision\",\n",
    "        context=\"/document\",\n",
    "        model_version=\"2023-04-15\",\n",
    "        inputs=[InputFieldMappingEntry(name=\"url\", source=\"/document/imageUrl\")],\n",
    "        outputs=[OutputFieldMappingEntry(name=\"vector\", target_name=\"imageVector\")],\n",
    "    )\n",
    "\n",
    "def create_skillset(client, skillset_name, text_embedding_skill, image_embedding_skill):\n",
    "    skillset = SearchIndexerSkillset(\n",
    "        name=skillset_name,\n",
    "        description=\"Skillset for generating embeddings\",\n",
    "        skills=[text_embedding_skill, image_embedding_skill],\n",
    "        cognitive_services_account=CognitiveServicesAccountKey(\n",
    "            key=AZURE_AI_VISION_API_KEY,\n",
    "            description=\"AI Vision Multi Service Account in West US\",\n",
    "        ),\n",
    "    )\n",
    "    client.create_or_update_skillset(skillset)\n",
    "\n",
    "client = SearchIndexerClient(\n",
    "    endpoint=SEARCH_SERVICE_ENDPOINT, credential=azure_search_credential\n",
    ")\n",
    "skillset_name = f\"{INDEX_NAME}-skillset\"\n",
    "text_embedding_skill = create_text_embedding_skill()\n",
    "image_embedding_skill = create_image_embedding_skill()\n",
    "\n",
    "create_skillset(client, skillset_name, text_embedding_skill, image_embedding_skill)\n",
    "print(f\"Created skillset: {skillset_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Indexer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build-multimodal-demo-indexer created or updated.\n",
      "build-multimodal-demo-indexer is running. If queries return no results, please wait a bit and try again.\n"
     ]
    }
   ],
   "source": [
    "def create_and_run_indexer(indexer_client, indexer_name, skillset_name, index_name, data_source_name):\n",
    "    indexer = SearchIndexer(\n",
    "        name=indexer_name,\n",
    "        description=\"Indexer to index documents and generate embeddings\",\n",
    "        skillset_name=skillset_name,\n",
    "        target_index_name=index_name,\n",
    "        data_source_name=data_source_name,\n",
    "        parameters=IndexingParameters(\n",
    "            configuration=IndexingParametersConfiguration(\n",
    "                parsing_mode=BlobIndexerParsingMode.JSON_ARRAY,\n",
    "                query_timeout=None,\n",
    "            ),\n",
    "        ),\n",
    "        field_mappings=[FieldMapping(source_field_name=\"id\", target_field_name=\"id\")],\n",
    "        output_field_mappings=[\n",
    "            FieldMapping(source_field_name=\"/document/captionVector\", target_field_name=\"captionVector\"),\n",
    "            FieldMapping(source_field_name=\"/document/imageVector\", target_field_name=\"imageVector\"),\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    indexer_client.create_or_update_indexer(indexer)\n",
    "    print(f\"{indexer_name} created or updated.\")\n",
    "\n",
    "    indexer_client.run_indexer(indexer_name)\n",
    "    print(f\"{indexer_name} is running. If queries return no results, please wait a bit and try again.\")\n",
    "\n",
    "indexer_client = SearchIndexerClient(\n",
    "    endpoint=SEARCH_SERVICE_ENDPOINT, credential=azure_search_credential\n",
    ")\n",
    "data_source_name = f\"{INDEX_NAME}-blob\"\n",
    "indexer_name = f\"{INDEX_NAME}-indexer\"\n",
    "\n",
    "create_and_run_indexer(indexer_client, indexer_name, skillset_name, INDEX_NAME, data_source_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple vector search (text to text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Caption: a woman in a red dress\n",
      "Score: 0.9404777\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture286.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a red handbag with a strap\n",
      "Score: 0.90242684\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture526.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a woman in a dress\n",
      "Score: 0.8996681\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture300.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Initialize the SearchClient\n",
    "search_client = SearchClient(\n",
    "    SEARCH_SERVICE_ENDPOINT,\n",
    "    index_name=INDEX_NAME,\n",
    "    credential=azure_search_credential,\n",
    ")\n",
    "\n",
    "# Define the query\n",
    "# query = \"sunglasses for holiday\"\n",
    "# query = \"休日のサングラス\" # Japanese query\n",
    "query = \"female red dresses\" # Spanish query\n",
    "\n",
    "vector_query = VectorizableTextQuery(\n",
    "    text=query,\n",
    "    k_nearest_neighbors=3,\n",
    "    fields=\"captionVector\",\n",
    "    # fields=\"imageVector\",\n",
    ")\n",
    "\n",
    "# Perform the search\n",
    "results = search_client.search(\n",
    "    search_text=None,\n",
    "    vector_queries=[vector_query],\n",
    "    top=3\n",
    ")\n",
    "\n",
    "\n",
    "# Print the results\n",
    "for result in results:\n",
    "    print(f\"Caption: {result['caption']}\")\n",
    "    print(f\"Score: {result['@search.score']}\")\n",
    "    display(HTML(f'<img src=\"{result[\"imageUrl\"]}\" style=\"width:200px;\"/>'))\n",
    "    print(\"-\" * 50) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Vector Search (text to image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Caption: a group of high heels\n",
      "Score: 0.032522473484277725\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture504.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture504.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a group of clothes and shoes\n",
      "Score: 0.01666666753590107\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture477.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture477.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a group of shoes on a wood surface\n",
      "Score: 0.01666666753590107\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture496.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture496.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Define the text query\n",
    "query = \"shoes for running\"\n",
    "text_vector_query = VectorizableTextQuery(\n",
    "    text=query,\n",
    "    k_nearest_neighbors=10,\n",
    "    fields=\"captionVector\",\n",
    ")\n",
    "# Define the image query\n",
    "image_vector_query = VectorizableImageUrlQuery(  # Alternatively, use VectorizableImageBinaryQuery\n",
    "    url=\"https://images.unsplash.com/photo-1542291026-7eec264c27ff?q=80&w=1770&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\",  # Image of a Red Nike Running Shoe\n",
    "    k_nearest_neighbors=10,\n",
    "    fields=\"imageVector\",\n",
    ")\n",
    "\n",
    "# Perform the search\n",
    "results = search_client.search(\n",
    "    search_text=None, vector_queries=[text_vector_query, image_vector_query], top=3\n",
    ")\n",
    "\n",
    "# Print the results\n",
    "for result in results:\n",
    "    print(f\"Caption: {result['caption']}\")\n",
    "    print(f\"Score: {result['@search.score']}\")\n",
    "    print(f\"URL: {result['imageUrl']}\")\n",
    "    display(HTML(f'<img src=\"{result[\"imageUrl\"]}\" style=\"width:200px;\"/>'))\n",
    "    print(\"-\" * 50)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-modal vector search with weighting images 100x more than captions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Caption: a group of shoes on a wood surface\n",
      "Score: 1.6666667461395264\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture496.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture496.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a woman in blue running\n",
      "Score: 1.6393442153930664\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture654.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture654.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Caption: a group of high heels\n",
      "Score: 1.6292965412139893\n",
      "URL: https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture504.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://fsunavalastorage.blob.core.windows.net/vector-sandbox/Picture504.jpg\" style=\"width:200px;\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Define the text query\n",
    "query = \"shoes for running\"\n",
    "text_vector_query = VectorizableTextQuery(\n",
    "    text=query,\n",
    "    k_nearest_neighbors=5,\n",
    "    fields=\"captionVector\",\n",
    ")\n",
    "# Define the image query\n",
    "image_vector_query = VectorizableImageUrlQuery(  # Alternatively, use VectorizableImageBinaryQuery\n",
    "    url=\"https://images.unsplash.com/photo-1542291026-7eec264c27ff?q=80&w=1770&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\",  # Image of a Red Nike Running Shoe\n",
    "    k_nearest_neighbors=5,\n",
    "    fields=\"imageVector\",\n",
    "    weight=100,\n",
    ")\n",
    "\n",
    "# Perform the search\n",
    "results = search_client.search(\n",
    "    search_text=None, vector_queries=[text_vector_query, image_vector_query], top=3\n",
    ")\n",
    "\n",
    "# Print the results\n",
    "for result in results:\n",
    "    print(f\"Caption: {result['caption']}\")\n",
    "    print(f\"Score: {result['@search.score']}\")\n",
    "    print(f\"URL: {result['imageUrl']}\")\n",
    "    display(HTML(f'<img src=\"{result[\"imageUrl\"]}\" style=\"width:200px;\"/>'))\n",
    "    print(\"-\" * 50)  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}