in src/dotnet/AzureAI.Proxy/ReverseProxy/RetryMiddleware.cs [62:117]
private DestinationState PickOneDestination(IReverseProxyFeature reverseProxyFeature)
{
var allDestinations = reverseProxyFeature.AllDestinations;
var selectedPriority = int.MaxValue;
var availableBackends = new List<int>();
for (var i = 0; i < allDestinations.Count; i++)
{
var destination = allDestinations[i];
if (destination.Health.Passive != DestinationHealth.Unhealthy)
{
var destinationPriority = int.Parse(destination.Model.Config.Metadata["priority"]);
if (destinationPriority < selectedPriority)
{
selectedPriority = destinationPriority;
availableBackends.Clear();
availableBackends.Add(i);
}
else if (destinationPriority == selectedPriority)
{
availableBackends.Add(i);
}
}
}
int backendIndex;
if (availableBackends.Count > 0)
{
if (availableBackends.Count == 1)
{
//Returns the only available backend if we have only one available
backendIndex = availableBackends[0];
}
else
{
//Returns a random backend from the list if we have more than one available with the same priority
backendIndex = availableBackends[Random.Shared.Next(0, availableBackends.Count)];
}
}
else
{
//Returns a random backend if all backends are unhealthy
_logger.LogWarning($"All backends are unhealthy. Picking a random backend...");
backendIndex = Random.Shared.Next(0, allDestinations.Count);
}
var pickedDestination = allDestinations[backendIndex];
_logger.LogInformation($"Picked backend: {pickedDestination.Model.Config.Address}");
return pickedDestination;
}