in setup.py [0:0]
def execute_setup(subscription_id, resource_group, function_app_name, search_principal_id, azure_search_use_mis, enable_managed_identities, enable_env_credentials):
"""
This function performs the necessary steps to set up the ingestion sub components.
"""
logging.info(f"Getting function app {function_app_name} properties.")
credential = ChainedTokenCredential(
ManagedIdentityCredential(),
AzureCliCredential()
)
web_mgmt_client = WebSiteManagementClient(credential, subscription_id)
function_app_settings = web_mgmt_client.web_apps.list_application_settings(resource_group, function_app_name)
function_endpoint = f"https://{function_app_name}.azurewebsites.net"
azure_openai_service_name = function_app_settings.properties["AZURE_OPENAI_SERVICE_NAME"]
search_service = function_app_settings.properties["AZURE_SEARCH_SERVICE"]
search_analyzer_name = function_app_settings.properties["SEARCH_ANALYZER_NAME"]
search_api_version = function_app_settings.properties.get("SEARCH_API_VERSION", "2024-07-01")
search_index_interval = function_app_settings.properties["SEARCH_INDEX_INTERVAL"]
search_index_name = function_app_settings.properties["SEARCH_INDEX_NAME"]
storage_container = function_app_settings.properties["STORAGE_CONTAINER"]
storage_account_name = function_app_settings.properties["STORAGE_ACCOUNT_NAME"]
network_isolation = True if function_app_settings.properties["NETWORK_ISOLATION"].lower() == "true" else False
storage_container = function_app_settings.properties["STORAGE_CONTAINER"]
storage_account_name = function_app_settings.properties["STORAGE_ACCOUNT_NAME"]
azure_openai_embedding_deployment = function_app_settings.properties.get("AZURE_OPENAI_EMBEDDING_DEPLOYMENT", "text-embedding")
azure_openai_embedding_model = function_app_settings.properties.get("AZURE_OPENAI_EMBEDDING_MODEL", "text-embedding-3-large")
azure_embeddings_vector_size = function_app_settings.properties.get("AZURE_EMBEDDINGS_VECTOR_SIZE", "3072")
azure_storage_resource_group = function_app_settings.properties["AZURE_STORAGE_ACCOUNT_RG"]
azure_aoai_resource_group = function_app_settings.properties["AZURE_AOAI_RG"]
logging.info(f"[execute_setup] Function endpoint: {function_endpoint}")
logging.info(f"[execute_setup] Search service: {search_service}")
logging.info(f"[execute_setup] Search analyzer name: {search_analyzer_name}")
logging.info(f"[execute_setup] Search API version: {search_api_version}")
logging.info(f"[execute_setup] Search index interval: {search_index_interval}")
logging.info(f"[execute_setup] Search index name: {search_index_name}")
logging.info(f"[execute_setup] Storage container: {storage_container}")
logging.info(f"[execute_setup] Storage account name: {storage_account_name}")
logging.info(f"[execute_setup] Embedding deployment name: {azure_openai_embedding_deployment}")
logging.info(f"[execute_setup] Embedding model: {azure_openai_embedding_model}")
logging.info(f"[execute_setup] Embedding vector size: {azure_embeddings_vector_size}")
logging.info(f"[execute_setup] Resource group: {resource_group}")
logging.info(f"[execute_setup] Storage resource group: {azure_storage_resource_group}")
logging.info(f"[execute_setup] Azure OpenAI resource group: {azure_aoai_resource_group}")
###########################################################################
# NL2SQL Elements
###########################################################################
storage_container_nl2sql = "nl2sql"
search_index_name_nl2sql_queries = "nl2sql-queries"
search_index_name_nl2sql_tables = "nl2sql-tables"
search_index_name_nl2sql_measures = "nl2sql-measures" # New measures index
logging.info(f"[execute_setup] NL2SQL Storage container: {storage_container_nl2sql}")
logging.info(f"[execute_setup] NL2SQL Search index name (queries): {search_index_name_nl2sql_queries}")
logging.info(f"[execute_setup] NL2SQL Search index name (tables): {search_index_name_nl2sql_tables}")
logging.info(f"[execute_setup] NL2SQL Search index name (measures): {search_index_name_nl2sql_measures}")
###########################################################################
# Get function key to be used later when creating the skillset
###########################################################################
function_key = get_function_key(subscription_id, resource_group, function_app_name, credential)
if function_key is None:
logging.error(f"Could not get function key. Please make sure the function {function_app_name}/document_chunking is deployed before running this script.")
exit(1)
###########################################################################
# Approve Search Shared Private Links (if needed)
###########################################################################
logging.info("Approving search shared private links.")
approve_search_shared_private_access(subscription_id, resource_group, azure_storage_resource_group, azure_aoai_resource_group, function_app_name, storage_account_name, azure_openai_service_name, credential)
###########################################################################
# Creating blob containers
###########################################################################
# Note: this step was removed since the storage account and container are already created by azd provision
###############################################################################
# Creating AI Search datasource
###############################################################################
def create_datasource(search_service, search_api_version, datasource_name, storage_connection_string, container_name, credential, subfolder=None):
body = {
"description": f"Datastore for {datasource_name}",
"type": "azureblob",
"dataDeletionDetectionPolicy": {
"@odata.type": "#Microsoft.Azure.Search.NativeBlobSoftDeleteDeletionDetectionPolicy"
},
"credentials": {
"connectionString": storage_connection_string
},
"container": {
"name": container_name,
"query": f"{subfolder}/" if subfolder else ""
}
}
call_search_api(search_service, search_api_version, "datasources", f"{datasource_name}-datasource", "put", credential, body)
logging.info("Creating datasources step.")
start_time = time.time()
storage_connection_string = f"ResourceId=/subscriptions/{subscription_id}/resourceGroups/{azure_storage_resource_group}/providers/Microsoft.Storage/storageAccounts/{storage_account_name}/;"
create_datasource(search_service, search_api_version, f"{search_index_name}", storage_connection_string, storage_container, credential)
nl2sql_subfolders = {
"queries": search_index_name_nl2sql_queries,
"tables": search_index_name_nl2sql_tables,
"measures": search_index_name_nl2sql_measures # New datasource for measures
}
for subfolder, index_name in nl2sql_subfolders.items():
create_datasource(search_service, search_api_version, index_name, storage_connection_string, "nl2sql", credential, subfolder=subfolder)
response_time = time.time() - start_time
logging.info(f"Create datastores step. {round(response_time, 2)} seconds")
###############################################################################
# Creating indexes
###############################################################################
def create_index_body(index_name, fields, content_fields_name, keyword_field_name, vector_profile_name="myHnswProfile", vector_algorithm_name="myHnswConfig"):
body = {
"name": index_name,
"fields": fields,
"corsOptions": {
"allowedOrigins": ["*"],
"maxAgeInSeconds": 60
},
"vectorSearch": {
"profiles": [
{
"name": vector_profile_name,
"algorithm": vector_algorithm_name
}
],
"algorithms": [
{
"name": vector_algorithm_name,
"kind": "hnsw",
"hnswParameters": {
"m": 4,
"efConstruction": 400,
"efSearch": 500,
"metric": "cosine"
}
}
]
},
"semantic": {
"configurations": [
{
"name": "my-semantic-config",
"prioritizedFields": {
"prioritizedContentFields": [
{
"fieldName": field_name
}
for field_name in content_fields_name
]
}
}
]
}
}
if keyword_field_name is not None:
body["semantic"]["configurations"][0]["prioritizedFields"]["prioritizedKeywordsFields"] = [
{
"fieldName": keyword_field_name
}
]
return body
logging.info("Creating indexes.")
start_time = time.time()
vector_profile_name = "myHnswProfile"
vector_algorithm_name = "myHnswConfig"
indices = [
{
"index_name": search_index_name, # RAG index
"fields": [
{
"name": "id",
"type": "Edm.String",
"key": True,
"analyzer": "keyword",
"searchable": True,
"retrievable": True
},
{
"name": "parent_id",
"type": "Edm.String",
"searchable": False,
"retrievable": True
},
{
"name": "metadata_storage_path",
"type": "Edm.String",
"searchable": False,
"sortable": False,
"filterable": False,
"facetable": False
},
{
"name": "metadata_storage_name",
"type": "Edm.String",
"searchable": False,
"sortable": False,
"filterable": False,
"facetable": False
},
{
"name": "metadata_storage_last_modified",
"type": "Edm.DateTimeOffset",
"searchable": False,
"sortable": True,
"retrievable": True,
"filterable": True
},
{
"name": "metadata_security_id",
"type": "Collection(Edm.String)",
"searchable": False,
"retrievable": True,
"filterable": True
},
{
"name": "chunk_id",
"type": "Edm.Int32",
"searchable": False,
"retrievable": True
},
{
"name": "content",
"type": "Edm.String",
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "imageCaptions",
"type": "Edm.String",
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "page",
"type": "Edm.Int32",
"searchable": False,
"retrievable": True
},
{
"name": "offset",
"type": "Edm.Int64",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "length",
"type": "Edm.Int32",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "title",
"type": "Edm.String",
"filterable": True,
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "category",
"type": "Edm.String",
"filterable": True,
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "filepath",
"type": "Edm.String",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "url",
"type": "Edm.String",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "summary",
"type": "Edm.String",
"filterable": False,
"searchable": True,
"retrievable": True
},
{
"name": "relatedImages",
"type": "Collection(Edm.String)",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "relatedFiles",
"type": "Collection(Edm.String)",
"filterable": False,
"searchable": False,
"retrievable": True
},
{
"name": "source",
"type": "Edm.String",
"searchable": False,
"retrievable": True,
"filterable": True
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"retrievable": True,
"dimensions": azure_embeddings_vector_size,
"vectorSearchProfile": vector_profile_name
},
{
"name": "captionVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"retrievable": True,
"dimensions": azure_embeddings_vector_size,
"vectorSearchProfile": vector_profile_name
}
],
"content_fields_name": ["content", "imageCaptions"],
"keyword_field_name": "category"
},
{
"index_name": search_index_name_nl2sql_queries,
"fields": [
{
"name": "id",
"type": "Edm.String",
"key": True,
"searchable": False,
"filterable": False,
"sortable": False,
"facetable": False
},
{
"name": "datasource",
"type": "Edm.String",
"searchable": True,
"filterable": True,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "question",
"type": "Edm.String",
"searchable": True,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False,
"analyzer": search_analyzer_name
},
{
"name": "query",
"type": "Edm.String",
"searchable": False,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "reasoning",
"type": "Edm.String",
"searchable": True,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"retrievable": True,
"dimensions": azure_embeddings_vector_size,
"vectorSearchProfile": vector_profile_name
}
],
"content_fields_name": ["question"],
"keyword_field_name": None
},
{
"index_name": search_index_name_nl2sql_tables,
"fields": [
{
"name": "id",
"type": "Edm.String",
"key": True,
"searchable": False,
"filterable": False,
"sortable": False,
"facetable": False
},
{
"name": "table",
"type": "Edm.String",
"searchable": True,
"retrievable": True
},
{
"name": "description",
"type": "Edm.String",
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "datasource",
"type": "Edm.String",
"searchable": True,
"retrievable": True
},
{
"name": "columns",
"type": "Collection(Edm.ComplexType)",
"fields": [
{
"name": "name",
"type": "Edm.String",
"searchable": True,
"retrievable": True
},
{
"name": "description",
"type": "Edm.String",
"searchable": True,
"retrievable": True,
"analyzer": search_analyzer_name
},
{
"name": "type",
"type": "Edm.String",
"searchable": False,
"retrievable": True
},
{
"name": "examples",
"type": "Collection(Edm.String)",
"searchable": False,
"retrievable": True
}
]
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"retrievable": True,
"dimensions": azure_embeddings_vector_size,
"vectorSearchProfile": vector_profile_name
}
],
"content_fields_name": ["description"],
"keyword_field_name": "table"
},
{
"index_name": search_index_name_nl2sql_measures,
"fields": [
{
"name": "id",
"type": "Edm.String",
"key": True,
"searchable": False,
"filterable": False,
"sortable": False,
"facetable": False
},
{
"name": "datasource",
"type": "Edm.String",
"searchable": True,
"filterable": True,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "name",
"type": "Edm.String",
"searchable": True,
"filterable": True,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "description",
"type": "Edm.String",
"searchable": True,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False,
"analyzer": search_analyzer_name
},
{
"name": "type",
"type": "Edm.String",
"searchable": True,
"filterable": True,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "source_table",
"type": "Edm.String",
"searchable": True,
"filterable": True,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "data_type",
"type": "Edm.String",
"searchable": True,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "source_model",
"type": "Edm.String",
"searchable": True,
"filterable": False,
"retrievable": True,
"sortable": False,
"facetable": False
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"retrievable": True,
"dimensions": azure_embeddings_vector_size,
"vectorSearchProfile": vector_profile_name
}
],
"content_fields_name": ["description"],
"keyword_field_name": "description"
}
]
for index in indices:
body = create_index_body(
index_name=index["index_name"],
fields=index["fields"],
content_fields_name=index["content_fields_name"],
keyword_field_name=index["keyword_field_name"],
vector_profile_name=vector_profile_name,
vector_algorithm_name=vector_algorithm_name
)
call_search_api(search_service, search_api_version, "indexes", index["index_name"], "delete", credential)
call_search_api(search_service, search_api_version, "indexes", index["index_name"], "put", credential, body)
response_time = time.time() - start_time
logging.info(f"Indexes created in {round(response_time, 2)} seconds")
###########################################################################
# 04 Creating AI Search skillsets
###########################################################################
logging.info("04 Creating skillsets step.")
start_time = time.time()
body = {
"name": f"{search_index_name}-skillset-chunking",
"description":"SKillset to do document chunking",
"skills":[
{
"@odata.type":"#Microsoft.Skills.Custom.WebApiSkill",
"name":"document-chunking",
"description":"Extract chunks from documents.",
"httpMethod":"POST",
"timeout":"PT230S",
"context":"/document",
"batchSize":1,
"inputs":[
{
"name":"documentUrl",
"source":"/document/metadata_storage_path"
},
{
"name":"documentSasToken",
"source":"/document/metadata_storage_sas_token"
},
{
"name":"documentContentType",
"source":"/document/metadata_content_type"
}
],
"outputs":[
{
"name":"chunks",
"targetName":"chunks"
}
]
}
],
"indexProjections": {
"selectors": [
{
"targetIndexName":f"{search_index_name}",
"parentKeyFieldName": "parent_id",
"sourceContext": "/document/chunks/*",
"mappings": [
{
"name": "chunk_id",
"source": "/document/chunks/*/chunk_id",
"inputs": []
},
{
"name": "offset",
"source": "/document/chunks/*/offset",
"inputs": []
},
{
"name": "length",
"source": "/document/chunks/*/length",
"inputs": []
},
{
"name": "page",
"source": "/document/chunks/*/page",
"inputs": []
},
{
"name": "title",
"source": "/document/chunks/*/title",
"inputs": []
},
{
"name": "category",
"source": "/document/chunks/*/category",
"inputs": []
},
{
"name": "url",
"source": "/document/chunks/*/url",
"inputs": []
},
{
"name": "relatedImages",
"source": "/document/chunks/*/relatedImages",
"inputs": []
},
{
"name": "relatedFiles",
"source": "/document/chunks/*/relatedFiles",
"inputs": []
},
{
"name": "filepath",
"source": "/document/chunks/*/filepath",
"inputs": []
},
{
"name": "content",
"source": "/document/chunks/*/content",
"inputs": []
},
{
"name": "imageCaptions",
"source": "/document/chunks/*/imageCaptions",
"inputs": []
},
{
"name": "summary",
"source": "/document/chunks/*/summary",
"inputs": []
},
{
"name": "source",
"source": "/document/chunks/*/source",
"inputs": []
},
{
"name": "captionVector",
"source": "/document/chunks/*/captionVector",
"inputs": []
},
{
"name": "contentVector",
"source": "/document/chunks/*/contentVector",
"inputs": []
},
{
"name": "metadata_storage_last_modified",
"source": "/document/metadata_storage_last_modified",
"inputs": []
},
{
"name": "metadata_storage_name",
"source": "/document/metadata_storage_name",
"inputs": []
},
{
"name": "metadata_storage_path",
"source": "/document/metadata_storage_path",
"inputs": []
},
{
"name": "metadata_security_id",
"source": "/document/metadata_security_id",
"inputs": []
}
]
}
],
"parameters": {
"projectionMode": "skipIndexingParentDocuments"
}
}
}
if azure_search_use_mis:
body['skills'][0]['uri'] = f"{function_endpoint}/api/document-chunking"
body['skills'][0]['authResourceId'] = f"api://{search_principal_id}"
else:
body['skills'][0]['uri'] = f"{function_endpoint}/api/document-chunking?code={function_key}"
call_search_api(search_service, search_api_version, "skillsets", f"{search_index_name}-skillset-chunking", "delete", credential)
call_search_api(search_service, search_api_version, "skillsets", f"{search_index_name}-skillset-chunking", "put", credential, body)
# creating skillsets for the NL2SQL indexes
def create_embedding_skillset(skillset_name, resource_uri, deployment_id, model_name, input_field, output_field, dimensions):
skill = {
"@odata.type": "#Microsoft.Skills.Text.AzureOpenAIEmbeddingSkill",
"name": f"{skillset_name}-embedding-skill",
"description": f"Generates embeddings for {input_field}.",
"resourceUri": resource_uri,
"deploymentId": deployment_id,
"modelName": model_name,
"dimensions": dimensions,
"context":"/document",
"inputs": [
{
"name": "text",
"source": f"/document/{input_field}"
}
],
"outputs": [
{
"name": "embedding",
"targetName": output_field
}
]
}
skillset_body = {
"name": skillset_name,
"description": f"Skillset for generating embeddings for {skillset_name} index.",
"skills": [skill]
}
return skillset_body
resource_uri = f"https://{azure_openai_service_name}.openai.azure.com/"
deployment_id = azure_openai_embedding_deployment
model_name = azure_openai_embedding_model
skillsets = [
{
"skillset_name": "queries-skillset",
"input_field": "question",
"output_field": "contentVector"
},
{
"skillset_name": "tables-skillset",
"input_field": "description",
"output_field": "contentVector"
},
{
"skillset_name": "measures-skillset", # New measures skillset
"input_field": "description",
"output_field": "contentVector"
}
]
for skillset in skillsets:
body = create_embedding_skillset(
skillset_name=skillset["skillset_name"],
resource_uri=resource_uri,
deployment_id=deployment_id,
model_name=model_name,
input_field=skillset["input_field"],
output_field=skillset["output_field"],
dimensions=azure_embeddings_vector_size
)
call_search_api(search_service, search_api_version, "skillsets", skillset["skillset_name"], "delete", credential)
call_search_api(search_service, search_api_version, "skillsets", skillset["skillset_name"], "put", credential, body)
logging.info(f"Skillset '{skillset['skillset_name']}' created successfully.")
response_time = time.time() - start_time
logging.info(f"04 Create skillset step. {round(response_time,2)} seconds")
###########################################################################
# 05 Creating indexers
###########################################################################
logging.info("05 Creating indexer step.")
start_time = time.time()
body = {
"dataSourceName" : f"{search_index_name}-datasource",
"targetIndexName" : f"{search_index_name}",
"skillsetName" : f"{search_index_name}-skillset-chunking",
"schedule" : { "interval" : f"{search_index_interval}"},
"fieldMappings" : [
{
"sourceFieldName" : "metadata_storage_path",
"targetFieldName" : "id",
"mappingFunction" : {
"name" : "fixedLengthEncode"
}
}
],
"outputFieldMappings" : [
],
"parameters":
{
"batchSize": 1,
"maxFailedItems":-1,
"maxFailedItemsPerBatch":-1,
"base64EncodeKeys": True,
"configuration":
{
"dataToExtract": "allMetadata"
}
}
}
if network_isolation:
body['parameters']['configuration']['executionEnvironment'] = "private"
call_search_api(search_service, search_api_version, "indexers", f"{search_index_name}-indexer-chunk-documents", "put", credential, body)
def create_indexer_body(indexer_name, index_name, data_source_name, skillset_name, field_mappings=None, indexing_parameters=None):
body = {
"name": indexer_name,
"dataSourceName": data_source_name,
"targetIndexName": index_name,
"skillsetName": skillset_name,
"schedule": {
"interval": "PT2H"
},
"fieldMappings": field_mappings if field_mappings else [],
"outputFieldMappings": [
{
"sourceFieldName": "/document/contentVector",
"targetFieldName": "contentVector"
}
],
"parameters":
{
"configuration": {
"parsingMode": "json"
}
}
}
if indexing_parameters:
body["parameters"] = indexing_parameters
return body
field_mappings_queries = [
{
"sourceFieldName" : "metadata_storage_path",
"targetFieldName" : "id",
"mappingFunction" : {
"name" : "fixedLengthEncode"
}
},
{
"sourceFieldName": "datasource",
"targetFieldName": "datasource"
},
{
"sourceFieldName": "question",
"targetFieldName": "question"
},
{
"sourceFieldName": "query",
"targetFieldName": "query"
},
{
"sourceFieldName": "reasoning",
"targetFieldName": "reasoning"
}
]
field_mappings_tables = [
{
"sourceFieldName": "metadata_storage_path",
"targetFieldName": "id",
"mappingFunction": {
"name": "fixedLengthEncode"
}
},
{
"sourceFieldName": "table",
"targetFieldName": "table"
},
{
"sourceFieldName": "description",
"targetFieldName": "description"
},
{
"sourceFieldName": "datasource",
"targetFieldName": "datasource"
},
{
"sourceFieldName": "columns",
"targetFieldName": "columns"
}
]
# New field mappings for the measures index
field_mappings_measures = [
{
"sourceFieldName": "metadata_storage_path",
"targetFieldName": "id",
"mappingFunction": {
"name": "fixedLengthEncode"
}
},
{
"sourceFieldName": "datasource",
"targetFieldName": "datasource"
},
{
"sourceFieldName": "name",
"targetFieldName": "name"
},
{
"sourceFieldName": "description",
"targetFieldName": "description"
},
{
"sourceFieldName": "type",
"targetFieldName": "type"
},
{
"sourceFieldName": "source_table",
"targetFieldName": "source_table"
},
{
"sourceFieldName": "data_type",
"targetFieldName": "data_type"
},
{
"sourceFieldName": "source_model",
"targetFieldName": "source_model"
}
]
indexing_parameters = {
"configuration": {
"parsingMode": "json"
}
}
indexers = [
{
"indexer_name": "queries-indexer",
"index_name": f"{search_index_name_nl2sql_queries}",
"data_source_name": f"{search_index_name_nl2sql_queries}-datasource",
"skillset_name": "queries-skillset",
"field_mappings": field_mappings_queries,
"indexing_parameters": indexing_parameters
},
{
"indexer_name": "tables-indexer",
"index_name": f"{search_index_name_nl2sql_tables}",
"data_source_name": f"{search_index_name_nl2sql_tables}-datasource",
"skillset_name": "tables-skillset",
"field_mappings": field_mappings_tables,
"indexing_parameters": indexing_parameters
},
{
"indexer_name": "measures-indexer", # New measures indexer
"index_name": f"{search_index_name_nl2sql_measures}",
"data_source_name": f"{search_index_name_nl2sql_measures}-datasource",
"skillset_name": "measures-skillset",
"field_mappings": field_mappings_measures,
"indexing_parameters": indexing_parameters
}
]
for indexer in indexers:
body = create_indexer_body(
indexer_name=indexer["indexer_name"],
index_name=indexer["index_name"],
data_source_name=indexer["data_source_name"],
skillset_name=indexer["skillset_name"],
field_mappings=indexer["field_mappings"]
)
call_search_api(search_service, search_api_version, "indexers", indexer["indexer_name"], "delete", credential)
call_search_api(search_service, search_api_version, "indexers", indexer["indexer_name"], "put", credential, body)
logging.info(f"Indexer '{indexer['indexer_name']}' created successfully.")
response_time = time.time() - start_time
logging.info(f"05 Create indexers step. {round(response_time,2)} seconds")