in src/CognitiveKioskUWP/MainPage.xaml.cs [512:643]
private async Task ProcessImage(SoftwareBitmap image)
{
try
{
Func<Task<Stream>> imageStreamCallback;
using (InMemoryRandomAccessStream stream = new InMemoryRandomAccessStream())
{
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, stream);
encoder.SetSoftwareBitmap(image);
await encoder.FlushAsync();
// Read the pixel bytes from the memory stream
using (var reader = new DataReader(stream.GetInputStreamAt(0)))
{
var bytes = new byte[stream.Size];
await reader.LoadAsync((uint)stream.Size);
reader.ReadBytes(bytes);
imageStreamCallback = () => Task.FromResult<Stream>(new MemoryStream(bytes));
}
}
Microsoft.Azure.CognitiveServices.Vision.ComputerVision.ComputerVisionClient visionClient = new Microsoft.Azure.CognitiveServices.Vision.ComputerVision.ComputerVisionClient(
new ApiKeyServiceClientCredentials(settings.ComputerVisionKey),
new System.Net.Http.DelegatingHandler[] { });
// Create a prediction endpoint, passing in the obtained prediction key
CustomVisionPredictionClient customVisionClient = new CustomVisionPredictionClient()
{
ApiKey = settings.CustomVisionKey,
Endpoint = $"https://{settings.CustomVisionRegion}.api.cognitive.microsoft.com"
};
Microsoft.Azure.CognitiveServices.Vision.Face.FaceClient faceClient = new Microsoft.Azure.CognitiveServices.Vision.Face.FaceClient(
new ApiKeyServiceClientCredentials(settings.FaceKey),
new System.Net.Http.DelegatingHandler[] { });
visionClient.Endpoint = settings.ComputerVisionEndpoint;
faceClient.Endpoint = settings.FaceEndpoint;
List<VisualFeatureTypes> features =
new List<VisualFeatureTypes>()
{
VisualFeatureTypes.Categories, VisualFeatureTypes.Description,
VisualFeatureTypes.Tags, VisualFeatureTypes.Faces, VisualFeatureTypes.Brands
};
// The list of Face attributes to return.
IList<FaceAttributeType> faceAttributes =
new FaceAttributeType[]
{
FaceAttributeType.Gender, FaceAttributeType.Age,
FaceAttributeType.Smile, FaceAttributeType.Emotion,
FaceAttributeType.Glasses, FaceAttributeType.Hair
};
try
{
if (!imageAnalysisRunning && DateTime.Now.Subtract(imageAnalysisLastDate).TotalMilliseconds > 1000)
{
imageAnalysisRunning = true;
_ = Task.Run(async () =>
{
ImageAnalysis analysis = await visionClient.AnalyzeImageInStreamAsync(await imageStreamCallback(), features);
ImagePrediction analysisCV = null;
try
{
analysisCV = await customVisionClient.DetectImageWithNoStoreAsync(new Guid(settings.CustomVisionProjectId), settings.CustomVisionIterationName, await imageStreamCallback());
}
catch (Exception)
{
// Throw away error
}
UpdateWithAnalysis(analysis, analysisCV);
imageAnalysisLastDate = DateTime.Now;
imageAnalysisRunning = false;
});
}
var analysisFace = await faceClient.Face.DetectWithStreamWithHttpMessagesAsync(await imageStreamCallback(), returnFaceId: true, returnFaceAttributes: faceAttributes);
imageWidth = image.PixelWidth;
imageHeight = image.PixelHeight;
facesControl.UpdateEvent(new CognitiveEvent() { Faces = analysisFace.Body, ImageWidth = image.PixelWidth, ImageHeight = image.PixelHeight });
if (analysisFace.Body.Count() > 0 && settings.DoFaceDetection)
{
var groups = await faceClient.PersonGroup.ListWithHttpMessagesAsync();
var group = groups.Body.FirstOrDefault(x => x.Name == settings.GroupName);
if (group != null)
{
var results = await faceClient.Face.IdentifyWithHttpMessagesAsync(analysisFace.Body.Select(x => x.FaceId.Value).ToArray(), group.PersonGroupId);
foreach (var identifyResult in results.Body)
{
var cand = identifyResult.Candidates.FirstOrDefault(x => x.Confidence > settings.FaceThreshold / 100d);
if (cand == null)
{
Console.WriteLine("No one identified");
}
else
{
// Get top 1 among all candidates returned
var candidateId = cand.PersonId;
var person = await faceClient.PersonGroupPerson.GetWithHttpMessagesAsync(group.PersonGroupId, candidateId);
tagsControl.UpdateEvent(new CognitiveEvent() { IdentifiedPerson = person.Body, IdentifiedPersonPrediction = cand.Confidence });
Console.WriteLine("Identified as {0}", person.Body.Name);
}
}
}
}
}
catch (Exception)
{
// Eat exception
}
}
catch (Exception)
{
// eat this exception too
}
}