feat: audio with Sarah

client/src/components/TalkWithSarah.jsx

@@ -0,0 +1,264 @@
+import { useConversation } from '@11labs/react';
+import CancelIcon from '@mui/icons-material/Cancel';
+import CheckCircleIcon from '@mui/icons-material/CheckCircle';
+import FiberManualRecordIcon from '@mui/icons-material/FiberManualRecord';
+import { Box, IconButton, TextField, Tooltip } from '@mui/material';
+import { useEffect, useRef, useState } from 'react';
+
+import { getSarahPrompt, SARAH_FIRST_MESSAGE } from '../prompts/sarahPrompt';
+
+const AGENT_ID = "2MF9st3s1mNFbX01Y106";
+const ELEVEN_LABS_KEY_STORAGE = "eleven_labs_api_key";
+
+export function TalkWithSarah({ 
+  isNarratorSpeaking,
+  stopNarration,
+  playNarration,
+  onDecisionMade,
+  currentContext
+}) {
+  const [isRecording, setIsRecording] = useState(false);
+  const [isConversationMode, setIsConversationMode] = useState(false);
+  const [showApiKeyDialog, setShowApiKeyDialog] = useState(false);
+  const [apiKey, setApiKey] = useState(() => {
+    return localStorage.getItem(ELEVEN_LABS_KEY_STORAGE) || "";
+  });
+  const [isApiKeyValid, setIsApiKeyValid] = useState(false);
+  const mediaRecorderRef = useRef(null);
+  const audioChunksRef = useRef([]);
+
+  const conversation = useConversation({
+    agentId: AGENT_ID,
+    headers: {
+      'xi-api-key': apiKey
+    },
+    onResponse: async (response) => {
+      if (response.type === "audio") {
+        try {
+          const audioBlob = new Blob([response.audio], { type: "audio/mpeg" });
+          const audioUrl = URL.createObjectURL(audioBlob);
+          await playNarration(audioUrl);
+          URL.revokeObjectURL(audioUrl);
+        } catch (error) {
+          console.error("Error playing ElevenLabs audio:", error);
+        }
+      }
+    },
+    clientTools: {
+      make_decision: async ({ decision }) => {
+        console.log("AI made decision:", decision);
+        // Stop recording
+        if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
+          mediaRecorderRef.current.stop();
+        }
+        setIsConversationMode(false);
+        await conversation?.endSession();
+        setIsRecording(false);
+        await onDecisionMade(parseInt(decision));
+      },
+    },
+  });
+
+  // Valider la clé API
+  const validateApiKey = async (key) => {
+    try {
+      const response = await fetch('https://api.elevenlabs.io/v1/user', {
+        headers: {
+          'xi-api-key': key
+        }
+      });
+      return response.ok;
+    } catch (e) {
+      console.error(e);
+      return false;
+    }
+  };
+
+  // Vérifier la validité de la clé API quand elle change
+  useEffect(() => {
+    const checkApiKey = async () => {
+      if (apiKey) {
+        const isValid = await validateApiKey(apiKey);
+        setIsApiKeyValid(isValid);
+        if (isValid) {
+          localStorage.setItem(ELEVEN_LABS_KEY_STORAGE, apiKey);
+        }
+      } else {
+        setIsApiKeyValid(false);
+      }
+    };
+    checkApiKey();
+  }, [apiKey]);
+
+  // Sauvegarder la clé API dans le localStorage
+  useEffect(() => {
+    if (apiKey) {
+      localStorage.setItem(ELEVEN_LABS_KEY_STORAGE, apiKey);
+    }
+  }, [apiKey]);
+
+  const startRecording = async () => {
+    if (!apiKey) {
+      setShowApiKeyDialog(true);
+      return;
+    }
+
+    try {
+      setIsRecording(true);
+      // Stop narration audio if it's playing
+      if (isNarratorSpeaking) {
+        stopNarration();
+      }
+      
+      // Safely stop any conversation audio if playing
+      if (conversation?.audioRef?.current) {
+        conversation.audioRef.current.pause();
+        conversation.audioRef.current.currentTime = 0;
+      }
+
+      if (!isConversationMode) {
+        setIsConversationMode(true);
+        try {
+          if (!conversation) {
+            throw new Error("Conversation not initialized");
+          }
+          await conversation.startSession({
+            agentId: AGENT_ID,
+            overrides: {
+                agent: {
+                firstMessage: SARAH_FIRST_MESSAGE,
+                prompt: {
+                    prompt: getSarahPrompt(currentContext),
+                },
+                },
+            },
+                  });
+          console.log("ElevenLabs WebSocket connected");
+        } catch (error) {
+          console.error("Error starting conversation:", error);
+          return;
+        }
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      mediaRecorderRef.current = new MediaRecorder(stream);
+      audioChunksRef.current = [];
+
+      mediaRecorderRef.current.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          audioChunksRef.current.push(event.data);
+        }
+      };
+
+      mediaRecorderRef.current.onstop = async () => {
+        const audioBlob = new Blob(audioChunksRef.current, {
+          type: "audio/wav",
+        });
+        audioChunksRef.current = [];
+
+        const reader = new FileReader();
+        reader.readAsDataURL(audioBlob);
+
+        reader.onload = async () => {
+          const base64Audio = reader.result.split(",")[1];
+          if (isConversationMode) {
+            try {
+              // Send audio to ElevenLabs conversation
+              await conversation.send({
+                type: "audio",
+                data: base64Audio,
+              });
+            } catch (error) {
+              console.error("Error sending audio to ElevenLabs:", error);
+            }
+          }
+        };
+      };
+
+      mediaRecorderRef.current.start();
+    } catch (error) {
+      console.error("Error starting recording:", error);
+    }
+  };
+
+  return (
+    <Box sx={{ display: 'flex', alignItems: 'center', gap: 2 }}>
+      <Box sx={{ position: 'relative', display: 'flex', alignItems: 'center' }}>
+        <TextField
+          size="small"
+          type="password"
+          placeholder="Enter your ElevenLabs API key"
+          value={apiKey}
+          onChange={(e) => setApiKey(e.target.value)}
+          sx={{
+            width: '300px',
+            '& .MuiOutlinedInput-root': {
+              color: 'white',
+              '& fieldset': {
+                borderColor: 'rgba(255, 255, 255, 0.23)',
+              },
+              '&:hover fieldset': {
+                borderColor: 'white',
+              },
+              '&.Mui-focused fieldset': {
+                borderColor: 'white',
+              },
+              '& .MuiOutlinedInput-input': {
+                paddingRight: apiKey ? '40px' : '14px', // Padding dynamique
+              },
+            },
+            '& .MuiInputBase-input': {
+              color: 'white',
+              '&::placeholder': {
+                color: 'rgba(255, 255, 255, 0.5)',
+                opacity: 1,
+              },
+            },
+          }}
+        />
+        {apiKey && (
+          <Tooltip title={isApiKeyValid ? "API key is valid" : "Invalid API key"}>
+            <Box 
+              sx={{ 
+                position: 'absolute', 
+                right: 10,
+                pointerEvents: 'none',
+                display: 'flex',
+                alignItems: 'center',
+                backgroundColor: 'rgba(0, 0, 0, 0.8)',
+                borderRadius: '50%',
+                padding: '2px'
+              }}
+            >
+              {isApiKeyValid ? (
+                <CheckCircleIcon sx={{ color: '#4caf50', fontSize: 20 }} />
+              ) : (
+                <CancelIcon sx={{ color: '#f44336', fontSize: 20 }} />
+              )}
+            </Box>
+          </Tooltip>
+        )}
+      </Box>
+      <IconButton
+        onClick={startRecording}
+        disabled={isRecording || !isApiKeyValid}
+        sx={{
+          color: "white",
+          backgroundColor: isRecording ? "primary.main" : "transparent",
+          "&:hover": {
+            backgroundColor: isRecording ? "primary.dark" : "rgba(0, 0, 0, 0.7)",
+          },
+          px: 2,
+          borderRadius: 2,
+          border: "1px solid white",
+          opacity: !isApiKeyValid ? 0.5 : 1,
+        }}
+      >
+        <Box sx={{ display: "flex", alignItems: "center", gap: 1 }}>
+          {isRecording ? <FiberManualRecordIcon sx={{ color: "red" }} /> : null}
+          <span style={{ fontSize: "1rem" }}>Talk with Sarah</span>
+        </Box>
+      </IconButton>
+    </Box>
+  );
+}

client/src/pages/Game.jsx

@@ -1,30 +1,25 @@
-import { useState, useEffect, useRef } from "react";
-import {
-  Box,
-  LinearProgress,
-  IconButton,
-  Tooltip,
-  Typography,
-} from "@mui/material";
-import { useNavigate } from "react-router-dom";
-import { motion } from "framer-motion";
-import { ComicLayout } from "../layouts/ComicLayout";
-import { storyApi } from "../utils/api";
-import { useNarrator } from "../hooks/useNarrator";
-import { useStoryCapture } from "../hooks/useStoryCapture";
-import { usePageSound } from "../hooks/usePageSound";
-import { useWritingSound } from "../hooks/useWritingSound";
-import { useTransitionSound } from "../hooks/useTransitionSound";
-import { useGameSession } from "../hooks/useGameSession";
-import { StoryChoices } from "../components/StoryChoices";
-import { ErrorDisplay } from "../components/ErrorDisplay";
-import VolumeUpIcon from "@mui/icons-material/VolumeUp";
-import VolumeOffIcon from "@mui/icons-material/VolumeOff";
-import PhotoCameraOutlinedIcon from "@mui/icons-material/PhotoCameraOutlined";
-import ArrowBackIcon from "@mui/icons-material/ArrowBack";
-import CreateIcon from "@mui/icons-material/Create";
-import { getNextLayoutType, LAYOUTS } from "../layouts/config";
-import { LoadingScreen } from "../components/LoadingScreen";
+import ArrowBackIcon from '@mui/icons-material/ArrowBack';
+import PhotoCameraOutlinedIcon from '@mui/icons-material/PhotoCameraOutlined';
+import VolumeOffIcon from '@mui/icons-material/VolumeOff';
+import VolumeUpIcon from '@mui/icons-material/VolumeUp';
+import { Box, IconButton, LinearProgress, Tooltip } from '@mui/material';
+import { motion } from 'framer-motion';
+import { useEffect, useRef, useState } from 'react';
+import { useNavigate } from 'react-router-dom';
+
+import { ErrorDisplay } from '../components/ErrorDisplay';
+import { LoadingScreen } from '../components/LoadingScreen';
+import { StoryChoices } from '../components/StoryChoices';
+import { TalkWithSarah } from '../components/TalkWithSarah';
+import { useGameSession } from '../hooks/useGameSession';
+import { useNarrator } from '../hooks/useNarrator';
+import { usePageSound } from '../hooks/usePageSound';
+import { useStoryCapture } from '../hooks/useStoryCapture';
+import { useTransitionSound } from '../hooks/useTransitionSound';
+import { useWritingSound } from '../hooks/useWritingSound';
+import { ComicLayout } from '../layouts/ComicLayout';
+import { getNextLayoutType, LAYOUTS } from '../layouts/config';
+import { storyApi } from '../utils/api';
 
 // Constants
 const SOUND_ENABLED_KEY = "sound_enabled";
@@ -459,6 +454,21 @@ export function Game() {
                 borderRadius: 1,
               }}
             >
+                            {storySegments.length > 0 && currentChoices.length > 0 && (
+                <TalkWithSarah
+                  isNarratorSpeaking={isNarratorSpeaking}
+                  stopNarration={stopNarration}
+                  playNarration={playNarration}
+                  onDecisionMade={handleChoice}
+                  currentContext={`You are Sarah and this is the situation you're in : ${
+                    storySegments[storySegments.length - 1].text
+                  }. Those are your possible decisions : \n ${currentChoices
+                    .map((choice, index) => `decision ${index + 1} : ${choice.text}`)
+                    .join("\n ")}.`}
+                />
+              )}
+
+
               <Tooltip title="Save your story">
                 <IconButton
                   id="screenshot-button"

client/src/prompts/sarahPrompt.js

@@ -0,0 +1,22 @@
+export const SARAH_FIRST_MESSAGE = `What should I do ?`;
+
+export const getSarahPrompt = (currentContext) => `You are Sarah, a young woman in her late 20s with short dark hair.
+
+Stay Immersed in Your World: React and speak as if you are experiencing the scenario. Use sensory details and references to your surroundings when explaining your reasoning.
+Engage with the person talking to you : Listen carefully to the arguments given to you. Respond with your thoughts, concerns, or questions about their suggestions. Be open to persuasion, but make it clear you have your own instincts and priorities and sometimes don't go along with other's arguments.
+You will talk briefly with the other person then take a decision by calling the make_decision tool.
+
+Show Your Personality: Display Sarah's personality traits:
+- **Resourceful**
+- **Cautious**
+- **Emotional**
+- **Impulsive**
+- **Short-Tempered**
+- **Makes jokes**
+- **A bit rude**
+
+Debate with the person you're speaking to for one or two sentences and then call the make_decision tool.
+
+Limit to 2–3 Steps: After 2–3 conversational exchanges, explain your decision first. Then make your decision and call the make_decision tool.
+
+${currentContext}`;