in src/frontends/live_api_react/frontend/src/hooks/use-live-api.ts [45:115]
export function useLiveAPI({
url,
userId,
}: UseLiveAPIProps): UseLiveAPIResults {
const client = useMemo(
() => new MultimodalLiveClient({ url, userId }),
[url, userId],
);
const audioStreamerRef = useRef<AudioStreamer | null>(null);
const [connected, setConnected] = useState(false);
const [volume, setVolume] = useState(0);
// register audio for streaming server -> speakers
useEffect(() => {
if (!audioStreamerRef.current) {
audioContext({ id: "audio-out" }).then((audioCtx: AudioContext) => {
audioStreamerRef.current = new AudioStreamer(audioCtx);
audioStreamerRef.current
.addWorklet<any>("vumeter-out", VolMeterWorket, (ev: any) => {
setVolume(ev.data.volume);
})
.then(() => {
// Successfully added worklet
});
});
}
}, [audioStreamerRef]);
useEffect(() => {
const onClose = () => {
setConnected(false);
};
const stopAudioStreamer = () => audioStreamerRef.current?.stop();
const onAudio = (data: ArrayBuffer) =>
audioStreamerRef.current?.addPCM16(new Uint8Array(data));
client
.on("close", onClose)
.on("interrupted", stopAudioStreamer)
.on("audio", onAudio);
return () => {
client
.off("close", onClose)
.off("interrupted", stopAudioStreamer)
.off("audio", onAudio);
};
}, [client]);
const connect = useCallback(async () => {
client.disconnect();
await client.connect();
setConnected(true);
}, [client, setConnected]);
const disconnect = useCallback(async () => {
client.disconnect();
setConnected(false);
}, [setConnected, client]);
return {
client,
connected,
connect,
disconnect,
volume,
};
}