Update app.py

app.py

@@ -5,6 +5,8 @@ import mediapipe as mp
 import joblib
 import pandas as pd
 from numpy.linalg import norm
+import matplotlib.pyplot as plt
+import os
 
 # Function to load the Random Forest model
 @st.cache_resource
@@ -33,18 +35,11 @@ mp_drawing = mp.solutions.drawing_utils
 
 # Function to normalize landmarks
 def normalize_landmarks(landmarks):
-    # Center the landmarks
     center = np.mean(landmarks, axis=0)
     landmarks_centered = landmarks - center
-
-    # Scale the landmarks to unit variance
     std_dev = np.std(landmarks_centered, axis=0)
     landmarks_normalized = landmarks_centered / std_dev
-
-    # Replace NaN values with 0 (in case of division by zero)
-    landmarks_normalized = np.nan_to_num(landmarks_normalized)
-
-    return landmarks_normalized
+    return np.nan_to_num(landmarks_normalized)
 
 # Function to calculate angles between landmarks
 def calculate_angles(landmarks):
@@ -57,51 +52,76 @@ def calculate_angles(landmarks):
             angles.extend([angle_x, angle_y])
     return angles
 
+# Function to process image and predict alphabet
+def process_and_predict(image_path):
+    image = cv2.imread(image_path)
+    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    results = hands.process(image_rgb)
+    
+    if results.multi_hand_landmarks:
+        landmarks = np.array([[lm.x, lm.y] for lm in results.multi_hand_landmarks[0].landmark])
+        landmarks_normalized = normalize_landmarks(landmarks)
+        angles = calculate_angles(landmarks_normalized)
+        
+        angle_columns = [f'angle_{i}' for i in range(len(angles))]
+        angles_df = pd.DataFrame([angles], columns=angle_columns)
+        
+        probabilities = model.predict_proba(angles_df)[0]
+        predicted_class = model.classes_[np.argmax(probabilities)]
+        
+        return predicted_class, probabilities, landmarks
+    
+    return None, None, None
+
+# Function to plot hand landmarks
+def plot_hand_landmarks(landmarks, title):
+    fig, ax = plt.subplots()
+    ax.scatter(landmarks[:, 0], landmarks[:, 1], c='blue')
+    for connection in mp.solutions.hands.HAND_CONNECTIONS:
+        start_idx = connection[0]
+        end_idx = connection[1]
+        ax.plot([landmarks[start_idx, 0], landmarks[end_idx, 0]],
+                [landmarks[start_idx, 1], landmarks[end_idx, 1]], 'r-')
+    ax.invert_yaxis()
+    ax.set_title(title)
+    ax.axis('off')
+    return fig
+
 # Streamlit app
 st.title("ASL Recognition App")
 
-# Upload image using Streamlit's file uploader
-uploaded_file = st.file_uploader("Upload an image of an ASL sign", type=["jpg", "jpeg", "png"])
+# Define available alphabets
+all_alphabets = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+excluded_alphabets = 'DMNPTUVXZ'
+available_alphabets = ''.join(set(all_alphabets) - set(excluded_alphabets))
 
-if uploaded_file is not None:
-    try:
-        # Read the image
-        image = cv2.imdecode(np.frombuffer(uploaded_file.read(), np.uint8), cv2.IMREAD_COLOR)
-        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-        
-        # Process the image and find hand landmarks
-        results = hands.process(image_rgb)
+# Multi-select for alphabets
+selected_alphabets = st.multiselect("Select alphabets to recognize:", list(available_alphabets))
+
+if selected_alphabets:
+    num_selected = len(selected_alphabets)
+    cols = min(3, num_selected)  # Maximum 3 columns
+    rows = (num_selected + cols - 1) // cols
+    
+    for i in range(0, num_selected, cols):
+        row_alphabets = selected_alphabets[i:i+cols]
+        cols_in_row = st.columns(len(row_alphabets))
         
-        if results.multi_hand_landmarks:
-            for hand_landmarks in results.multi_hand_landmarks:
-                # Draw landmarks on the image
-                mp_drawing.draw_landmarks(image, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS)
-                
-                # Extract and normalize landmarks
-                landmarks = np.array([[lm.x, lm.y] for lm in hand_landmarks.landmark])
-                landmarks_normalized = normalize_landmarks(landmarks)
-                
-                # Calculate angles using normalized landmarks
-                angles = calculate_angles(landmarks_normalized)
-                
-                # Prepare input with feature names
-                angle_columns = [f'angle_{i}' for i in range(len(angles))]
-                angles_df = pd.DataFrame([angles], columns=angle_columns)
-                
-                # Predict the alphabet
-                probabilities = model.predict_proba(angles_df)[0]
-                top_indices = np.argsort(probabilities)[::-1][:5]
-                top_probabilities = probabilities[top_indices]
-                top_classes = model.classes_[top_indices]
-                
-                # Display the top 5 predictions
-                st.write("Top 5 Predicted Alphabets:")
-                for i in range(5):
-                    st.write(f"{top_classes[i]}: {top_probabilities[i]:.2f}")
-            
-            # Display the image with landmarks
-            st.image(image, caption="Processed Image with Landmarks", use_column_width=True)
-        else:
-            st.write("No hands detected. Please try another image.")
-    except Exception as e:
-        st.error(f"Error processing image: {e}")
+        for j, alphabet in enumerate(row_alphabets):
+            with cols_in_row[j]:
+                image_path = f'asl test set/{alphabet}.jpg'
+                if os.path.exists(image_path):
+                    predicted_class, probabilities, landmarks = process_and_predict(image_path)
+                    if predicted_class is not None:
+                        st.write(f"Alphabet: {alphabet}")
+                        st.write(f"Predicted: {predicted_class}")
+                        st.write(f"Confidence: {probabilities[model.classes_.tolist().index(predicted_class)]:.2f}")
+                        fig = plot_hand_landmarks(landmarks, f"Hand Landmarks for {alphabet}")
+                        st.pyplot(fig)
+                    else:
+                        st.write(f"No hand detected for {alphabet}")
+                else:
+                    st.write(f"Image not found for {alphabet}")
+
+else:
+    st.write("Please select at least one alphabet to recognize.")