in src/WebJobs.Extensions.OpenAI.Kusto/KustoSearchProvider.cs [56:95]
public async Task AddDocumentAsync(SearchableDocument document, CancellationToken cancellationToken)
{
(IKustoIngestClient kustoIngestClient, KustoConnectionStringBuilder connectionStringBuilder) =
this.kustoIngestClients.GetOrAdd(
document.ConnectionInfo!.ConnectionName,
name =>
{
KustoConnectionStringBuilder connectionStringBuilder = this.GetKustoConnectionString(name);
// NOTE: There are some scalability limits with the ingest client. Some scenarios
// may require a queued client.
IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionStringBuilder);
return (client, connectionStringBuilder);
});
string databaseName = connectionStringBuilder.InitialCatalog;
string tableName = document.ConnectionInfo.CollectionName;
DataTable table = new(tableName);
table.AppendColumn("Id", typeof(string));
table.AppendColumn("Title", typeof(string));
table.AppendColumn("Text", typeof(string));
table.AppendColumn("Embeddings", typeof(object));
table.AppendColumn("Timestamp", typeof(DateTime));
for (int i = 0; i < document.Embeddings?.Response?.Count; i++)
{
table.Rows.Add(
Guid.NewGuid().ToString("N"),
Path.GetFileNameWithoutExtension(document.Title),
document.Embeddings.Request![i],
GetEmbeddingsString(document.Embeddings.Response[i].ToFloats().ToArray(), true),
DateTime.UtcNow);
}
using IDataReader tableReader = table.CreateDataReader();
await kustoIngestClient.IngestFromDataReaderAsync(
tableReader,
new KustoIngestionProperties(databaseName, tableName));
}