in interactive-visualizers/src/app/app.component.ts [855:1011]
async runObjectDetector(image: HTMLImageElement, index: number):
Promise<void> {
const results = [];
if (this.modelFormat === 'tflite') {
const startTs = Date.now();
const detections = this.tfWebApi.run(image).getDetectionsList();
this.resultsLatency = Date.now() - startTs;
for (let i = 0; i < detections.length; i++) {
const detection = detections[i];
const boundingBox = detection.getBoundingBox();
const top = boundingBox.getOriginY() / image.height;
const left = boundingBox.getOriginX() / image.width;
const bottom = (boundingBox.getOriginY() + boundingBox.getHeight()) / image.height;
const right = (boundingBox.getOriginX() + boundingBox.getWidth()) / image.width;
let displayName = detection.getClassesList()[0].getDisplayName();
if (!displayName) {
displayName = detection.getClassesList()[0].getClassName();
}
results.push({
id: i,
box: [top, left, bottom, right],
score: detection.getClassesList()[0].getScore(),
label: detection.getClassesList()[0].getClassName(),
displayName,
});
}
} else {
// Prepare inputs.
const inputTensorMetadata =
this.modelMetadata.tfjs_detector_model_metadata.input_tensor_metadata;
const imageTensor = this.prepareImageInput(image, inputTensorMetadata);
// Execute the model.
const outputHeadMetadata =
this.modelMetadata.tfjs_detector_model_metadata.output_head_metadata[0];
const numDetectionsTensorName =
outputHeadMetadata.num_detections_tensor_name;
const detectionBoxesTensorName =
outputHeadMetadata.detection_boxes_tensor_name;
const detectionScoresTensorName =
outputHeadMetadata.detection_scores_tensor_name;
const detectionClassesTensorName =
outputHeadMetadata.detection_classes_tensor_name;
const startTs = Date.now();
const outputTensors = await this.model.executeAsync(imageTensor, [
numDetectionsTensorName, detectionBoxesTensorName,
detectionScoresTensorName, detectionClassesTensorName
]) as tf.Tensor[];
this.resultsLatency = Date.now() - startTs;
tf.dispose(imageTensor);
const squeezedNumDetections = await outputTensors[0].squeeze();
const squeezedDetectionBoxes = await outputTensors[1].squeeze();
const squeezedDetectionScores = await outputTensors[2].squeeze();
const squeezedDetectionClasses = await outputTensors[3].squeeze();
tf.dispose(outputTensors);
const numDetections = await squeezedNumDetections.array() as number;
const detectionBoxes = await squeezedDetectionBoxes.array() as number[][];
const detectionScores = await squeezedDetectionScores.array() as number[];
const detectionClasses = await squeezedDetectionClasses.array() as number[];
tf.dispose(squeezedNumDetections);
tf.dispose(squeezedDetectionBoxes);
tf.dispose(squeezedDetectionScores);
tf.dispose(squeezedDetectionClasses);
// Fetch labelmap and score thresholds.
this.detectionScoreThreshold = DEFAULT_DETECTION_THRESHOLD;
if (outputHeadMetadata.score_threshold != null &&
outputHeadMetadata.score_threshold.length) {
this.detectionScoreThreshold = outputHeadMetadata.score_threshold[0];
}
if (this.labelmap == null && outputHeadMetadata.labelmap_path != null) {
await this.fetchLabelmap(outputHeadMetadata.labelmap_path);
}
for (let i = 0; i < numDetections; ++i) {
const label = detectionClasses[i];
if (this.labelmap != null && this.labelmap.length > label) {
results.push({
id: i,
box: detectionBoxes[i],
score: detectionScores[i],
label,
displayName: this.labelmap[label],
});
} else {
results.push({
id: i,
box: detectionBoxes[i],
score: detectionScores[i],
label,
displayName: label,
});
}
}
}
// Display results only for the last selected image (as the user may
// have switched selection while inference was running).
if (this.imageSelectedIndex !== index) {
return;
}
// Prepare detector results general variables.
this.detectorResults = results;
this.resultsKeyName = 'Type';
this.resultsValueName = 'Score';
this.detectionLabelToIds.clear();
for (const result of results) {
if (!this.detectionLabelToIds.has(result.label)) {
const newId = this.detectionLabelToIds.size;
this.detectionLabelToIds.set(result.label, newId);
}
}
this.hoveredDetectionResultLabel = null;
this.hoveredDetectionResultId = null;
this.hoveredDetectionLabel = null;
this.hoveredDetectionId = null;
this.collapsedDetectionLabels = new Set();
// Prepare the result list content.
const labelDisplayNames = new Map();
const labelBoxes = new Map();
for (const result of results) {
if (!labelDisplayNames.has(result.label)) {
labelDisplayNames.set(result.label, result.displayName);
labelBoxes.set(result.label, []);
}
if (!this.collapsedDetectionLabels.has(result.label)) {
labelBoxes.get(result.label).push({
id: result.id,
score: result.score,
rect: result.box,
});
}
}
const labels = [];
for (const label of Array.from(labelDisplayNames.keys())) {
const colorIndex = this.detectionLabelToIds.get(label);
labels.push({
label,
displayName: labelDisplayNames.get(label),
boxes: labelBoxes.get(label),
color: `rgb(${255 * COLOR_LIST[colorIndex][0]}, ${
255 *
COLOR_LIST[colorIndex][1]}, ${255 * COLOR_LIST[colorIndex][2]})`,
});
}
this.detectionLabels = labels;
const imageHtmlElement = document.getElementById('query-image') as HTMLImageElement;
this.queryImageHeight = imageHtmlElement.offsetHeight;
this.queryImageWidth = imageHtmlElement.offsetWidth;
// Update score threshold position after letting time for the UI to update.
setTimeout(() => this.updateDetectionScoreThresholdPosition(), 20);
}