in src/WebJobs.Extensions.OpenAI.Kusto/KustoSearchProvider.cs [97:156]
public Task<SearchResponse> SearchAsync(SearchRequest request)
{
(ICslQueryProvider kustoQueryClient, KustoConnectionStringBuilder connectionStringBuilder) =
this.kustoQueryClients.GetOrAdd(
request.ConnectionInfo.ConnectionName,
name =>
{
KustoConnectionStringBuilder connectionStringBuilder = this.GetKustoConnectionString(name);
ICslQueryProvider client = KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder);
return (client, connectionStringBuilder);
});
// TODO: Use query parameters to remove the possibility of KQL-injection:
// https://learn.microsoft.com/azure/data-explorer/kusto/query/queryparametersstatement
// NOTE: Vector similarity reference:
// https://techcommunity.microsoft.com/t5/azure-data-explorer-blog/azure-data-explorer-for-vector-similarity-search/ba-p/3819626
string embeddingsList = GetEmbeddingsString(request.Embeddings, false);
string? tableName = request.ConnectionInfo.CollectionName?.Trim();
if (string.IsNullOrEmpty(tableName) ||
tableName.Contains('/') ||
tableName.Contains(';') ||
tableName.Any(char.IsWhiteSpace))
{
throw new InvalidOperationException($"The table name '{tableName}' is invalid.");
}
string query = $$"""
let series_cosine_similarity_fl=(vec1:dynamic, vec2:dynamic, vec1_size:real=double(null), vec2_size:real=double(null))
{
let dp = series_dot_product(vec1, vec2);
let v1l = iff(isnull(vec1_size), sqrt(series_dot_product(vec1, vec1)), vec1_size);
let v2l = iff(isnull(vec2_size), sqrt(series_dot_product(vec2, vec2)), vec2_size);
dp/(v1l*v2l)
};
let search_vector=pack_array({{embeddingsList}});
{{tableName}}
| extend similarity = series_cosine_similarity_fl(search_vector, Embeddings)
| top {{request.MaxResults}} by similarity desc
| project similarity, Id, Title, Text, Embeddings, Timestamp
""";
this.logger.LogDebug("Executing Kusto query: {query}", query);
using IDataReader reader = kustoQueryClient.ExecuteQuery(
databaseName: connectionStringBuilder.InitialCatalog,
query,
new ClientRequestProperties());
List<SearchResult> results = new(capacity: request.MaxResults);
while (reader.Read())
{
string subject = (string)reader["Title"];
string text = (string)reader["Text"];
results.Add(new SearchResult(subject, text));
}
this.logger.LogDebug("Kusto similarity query returned {count} results", results.Count);
SearchResponse response = new(results);
return Task.FromResult(response);
}