in src/frontends/live_api_react/frontend/src/utils/multimodal-live-client.ts [177:247]
protected async receive(blob: Blob) {
const response = (await blobToJSON(blob)) as LiveIncomingMessage;
console.log("Parsed response:", response);
if (isToolCallMessage(response)) {
this.log("server.toolCall", response);
this.emit("toolcall", response.toolCall);
return;
}
if (isToolCallCancellationMessage(response)) {
this.log("receive.toolCallCancellation", response);
this.emit("toolcallcancellation", response.toolCallCancellation);
return;
}
if (isSetupCompleteMessage(response)) {
this.log("server.send", "setupComplete");
this.emit("setupcomplete");
return;
}
// this json also might be `contentUpdate { interrupted: true }`
// or contentUpdate { end_of_turn: true }
if (isServerContenteMessage(response)) {
const { serverContent } = response;
if (isInterrupted(serverContent)) {
this.log("receive.serverContent", "interrupted");
this.emit("interrupted");
return;
}
if (isTurnComplete(serverContent)) {
this.log("server.send", "turnComplete");
this.emit("turncomplete");
//plausible there's more to the message, continue
}
if (isModelTurn(serverContent)) {
let parts: Part[] = serverContent.modelTurn.parts;
// when its audio that is returned for modelTurn
const audioParts = parts.filter(
(p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
);
const base64s = audioParts.map((p) => p.inlineData?.data);
// strip the audio parts out of the modelTurn
const otherParts = difference(parts, audioParts);
// console.log("otherParts", otherParts);
base64s.forEach((b64) => {
if (b64) {
const data = base64ToArrayBuffer(b64);
this.emit("audio", data);
this.log(`server.audio`, `buffer (${data.byteLength})`);
}
});
if (!otherParts.length) {
return;
}
parts = otherParts;
const content: ModelTurn = { modelTurn: { parts } };
this.emit("content", content);
this.log(`server.content`, response);
}
} else {
console.log("received unmatched message", response);
this.log("received unmatched message", response);
}
}