mesop-jeopardy-live / web_components /gemini_live_connection.js
Richard
Initial commit
09ed935
import {
LitElement,
html,
} from "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js";
class GeminiLiveConnection extends LitElement {
static properties = {
api_config: { type: String },
enabled: { type: Boolean },
endpoint: { type: String },
startEvent: { type: String },
stopEvent: { type: String },
text_input: { type: String },
toolCallEvent: { type: String },
tool_call_responses: { type: String },
};
constructor() {
super();
this.onSetupComplete = () => {
console.log("Setup complete...");
};
this.onAudioData = (base64Data) => {
this.dispatchEvent(
new CustomEvent("audio-output-received", {
detail: { data: base64Data },
// Allow event to cross shadow DOM boundaries (both need to be true)
bubbles: true,
composed: true,
})
);
};
this.onInterrupted = () => {};
this.onTurnComplete = () => {};
this.onError = () => {};
this.onClose = () => {
console.log("Web socket closed...");
};
this.onToolCall = (toolCalls) => {
this.dispatchEvent(
new MesopEvent(this.toolCallEvent, {
toolCalls: JSON.stringify(toolCalls.functionCalls),
})
);
};
this.pendingSetupMessage = null;
this.onAudioInputReceived = (e) => {
this.sendAudioChunk(e.detail.data);
};
}
connectedCallback() {
super.connectedCallback();
// Start listening for events when component is connected
window.addEventListener("audio-input-received", this.onAudioInputReceived);
}
disconnectedCallback() {
super.disconnectedCallback();
window.removeEventListener(
"audio-input-received",
this.onAudioInputReceived
);
if (this.ws) {
this.ws.close();
}
}
firstUpdated() {
if (this.enabled) {
this.setupWebSocket();
}
}
updated(changedProperties) {
if (
changedProperties.has("tool_call_responses") &&
this.tool_call_responses.length > 0
) {
this.sendToolResponse(JSON.parse(this.tool_call_responses));
}
if (changedProperties.has("text_input") && this.text_input.length > 0) {
this.sendTextMessage(this.text_input);
}
}
start() {
if (!this.enabled) {
this.dispatchEvent(new MesopEvent(this.startEvent, {}));
this.dispatchEvent(
new CustomEvent("gemini-live-api-started", {
detail: {},
// Allow event to cross shadow DOM boundaries (both need to be true)
bubbles: true,
composed: true,
})
);
}
this.setupWebSocket();
}
stop() {
this.dispatchEvent(new MesopEvent(this.stopEvent, {}));
this.dispatchEvent(
new CustomEvent("gemini-live-api-stopped", {
detail: {},
// Allow event to cross shadow DOM boundaries (both need to be true)
bubbles: true,
composed: true,
})
);
if (this.ws) {
this.ws.close();
}
}
setupWebSocket() {
this.ws = new WebSocket(this.endpoint);
this.ws.onopen = () => {
console.log("WebSocket connection is opening...");
this.sendSetupMessage();
};
this.ws.onmessage = async (event) => {
try {
let wsResponse;
if (event.data instanceof Blob) {
const responseText = await event.data.text();
wsResponse = JSON.parse(responseText);
} else {
wsResponse = JSON.parse(event.data);
}
if (wsResponse.setupComplete) {
this.onSetupComplete();
} else if (wsResponse.toolCall) {
this.onToolCall(wsResponse.toolCall);
} else if (wsResponse.serverContent) {
if (wsResponse.serverContent.interrupted) {
this.onInterrupted();
return;
}
if (wsResponse.serverContent.modelTurn?.parts?.[0]?.inlineData) {
const audioData =
wsResponse.serverContent.modelTurn.parts[0].inlineData.data;
this.onAudioData(audioData);
if (!wsResponse.serverContent.turnComplete) {
this.sendContinueSignal();
}
}
if (wsResponse.serverContent.turnComplete) {
this.onTurnComplete();
}
}
} catch (error) {
console.error("Error parsing response:", error);
this.onError("Error parsing response: " + error.message);
}
};
this.ws.onerror = (error) => {
console.error("WebSocket Error:", error);
this.onError("WebSocket Error: " + error.message);
};
this.ws.onclose = (event) => {
console.log("Connection closed:", event);
this.onClose(event);
};
}
sendMessage(message) {
if (this.ws.readyState === WebSocket.OPEN) {
this.ws.send(JSON.stringify(message));
} else {
console.error(
"WebSocket is not open. Current state:",
this.ws.readyState
);
this.onError("WebSocket is not ready. Please try again.");
}
}
sendSetupMessage() {
if (this.ws.readyState === WebSocket.OPEN) {
this.ws.send(this.api_config);
} else {
console.error("Connection not ready.");
}
}
sendAudioChunk(base64Audio) {
const message = {
realtime_input: {
media_chunks: [
{
mime_type: "audio/pcm",
data: base64Audio,
},
],
},
};
this.sendMessage(message);
}
sendEndMessage() {
const message = {
client_content: {
turns: [
{
role: "user",
parts: [],
},
],
turn_complete: true,
},
};
this.sendMessage(message);
}
sendContinueSignal() {
const message = {
client_content: {
turns: [
{
role: "user",
parts: [],
},
],
turn_complete: false,
},
};
this.sendMessage(message);
}
sendTextMessage(text) {
this.sendMessage({
client_content: {
turn_complete: true,
turns: [{ role: "user", parts: [{ text: text }] }],
},
});
}
sendToolResponse(functionResponses) {
const toolResponse = {
tool_response: {
function_responses: functionResponses,
},
};
this.sendMessage(toolResponse);
}
async ensureConnected() {
if (this.ws.readyState === WebSocket.OPEN) {
return;
}
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Connection timeout"));
}, 5000);
const onOpen = () => {
clearTimeout(timeout);
this.ws.removeEventListener("open", onOpen);
this.ws.removeEventListener("error", onError);
resolve();
};
const onError = (error) => {
clearTimeout(timeout);
this.ws.removeEventListener("open", onOpen);
this.ws.removeEventListener("error", onError);
reject(error);
};
this.ws.addEventListener("open", onOpen);
this.ws.addEventListener("error", onError);
});
}
render() {
if (this.enabled) {
return html`<span @click="${this.stop}"><slot></slot></span>`;
} else {
return html`<span @click="${this.start}"><slot></slot></span>`;
}
}
}
customElements.define("gemini-live-connection", GeminiLiveConnection);