app.py

import streamlit as st
import cv2
import numpy as np
import mediapipe as mp
import joblib
import pandas as pd
from numpy.linalg import norm
import matplotlib.pyplot as plt
import os

st.set_page_config(layout="wide")

# Define the alphabets
all_alphabets = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
excluded_alphabets = 'DMNPTUVXZ'
working_alphabets = ''.join(set(all_alphabets) - set(excluded_alphabets))

# Function to load the Random Forest model
@st.cache_resource
def load_model():
    """Load the pre-trained Random Forest model.

    Returns:
        model: The loaded Random Forest model.
    """
    try:
        return joblib.load('best_random_forest_model.pkl')
    except Exception as e:
        st.error(f"Error loading model: {e}")
        return None

# Load the model using the cached function
model = load_model()

# Ensure the model is loaded before proceeding
if model is None:
    st.stop()

# Function to normalize landmarks
def normalize_landmarks(landmarks):
    """Normalize the landmark coordinates.

    Args:
        landmarks (np.ndarray): The array of landmark coordinates.

    Returns:
        np.ndarray: Normalized landmark coordinates.
    """
    center = np.mean(landmarks, axis=0)
    landmarks_centered = landmarks - center
    std_dev = np.std(landmarks_centered, axis=0)
    landmarks_normalized = landmarks_centered / std_dev
    return np.nan_to_num(landmarks_normalized)

# Function to calculate angles between landmarks
def calculate_angles(landmarks):
    """Calculate angles between hand landmarks.

    Args:
        landmarks (np.ndarray): The array of normalized landmark coordinates.

    Returns:
        list: List of angles between landmarks.
    """
    angles = []
    for i in range(20):
        for j in range(i + 1, 21):
            vector = landmarks[j] - landmarks[i]
            angle_x = np.arccos(np.clip(vector[0] / norm(vector), -1.0, 1.0))
            angle_y = np.arccos(np.clip(vector[1] / norm(vector), -1.0, 1.0))
            angles.extend([angle_x, angle_y])
    return angles

# Function to process image and predict alphabet
def process_and_predict(image):
    """Process the uploaded image to predict the ASL sign.

    Args:
        image (np.ndarray): The uploaded image in BGR format.

    Returns:
        tuple: (probabilities of each class, detected landmarks)
    """
    mp_hands = mp.solutions.hands
    with mp_hands.Hands(static_image_mode=True, max_num_hands=1, min_detection_confidence=0.5) as hands:
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        results = hands.process(image_rgb)
        
        if results.multi_hand_landmarks:
            landmarks = np.array([[lm.x, lm.y] for lm in results.multi_hand_landmarks[0].landmark])
            landmarks_normalized = normalize_landmarks(landmarks)
            angles = calculate_angles(landmarks_normalized)
            
            angle_columns = [f'angle_{i}' for i in range(len(angles))]
            angles_df = pd.DataFrame([angles], columns=angle_columns)
            
            probabilities = model.predict_proba(angles_df)[0]
            return probabilities, landmarks
    
    return None, None

# Function to plot hand landmarks
def plot_hand_landmarks(landmarks, title):
    """Plot the detected hand landmarks.

    Args:
        landmarks (np.ndarray): The array of landmark coordinates.
        title (str): The title for the plot.

    Returns:
        Figure: The matplotlib figure object with plotted landmarks.
    """
    fig, ax = plt.subplots(figsize=(5, 5))
    ax.scatter(landmarks[:, 0], landmarks[:, 1], c='blue', s=20)
    mp_hands = mp.solutions.hands
    for connection in mp_hands.HAND_CONNECTIONS:
        start_idx = connection[0]
        end_idx = connection[1]
        ax.plot([landmarks[start_idx, 0], landmarks[end_idx, 0]],
                [landmarks[start_idx, 1], landmarks[end_idx, 1]], 'r-', linewidth=1)
    ax.invert_yaxis()
    ax.set_title(title, fontsize=12)
    ax.axis('off')
    return fig

# README content
readme_content = f"""
## How it works

This ASL Recognition App uses image processing and machine learning to recognize American Sign Language (ASL) hand signs.

1. **Image Upload**: Users can upload an image of an ASL hand sign.
2. **Hand Detection**: The app uses MediaPipe to detect hand landmarks in the image.
3. **Feature Extraction**: Angles between hand landmarks are calculated and normalized.
4. **Prediction**: A Random Forest model predicts the ASL sign based on the extracted features.
5. **Visualization**: The app displays the detected hand landmarks and top predictions.

### Supported Alphabets

The app currently works for the following ASL alphabets:
{', '.join(working_alphabets)}

The app does not support or may not work correctly for:
{', '.join(excluded_alphabets)}

Note: The model's performance may vary and is subject to improvement.

The "View Hand Landmarks" tab allows users to see hand landmarks for pre-loaded ASL signs.
"""

# Streamlit app
st.title("ASL Recognition App")

# Display README content
st.sidebar.markdown(readme_content)

# Create tabs for different functionalities
tab1, tab2 = st.tabs(["Predict ASL Sign", "View Hand Landmarks"])

with tab1:
    st.header("Predict ASL Sign")
    uploaded_file = st.file_uploader("Upload an image of an ASL sign", type=["jpg", "jpeg", "png"])

    if uploaded_file is not None:
        try:
            image = cv2.imdecode(np.frombuffer(uploaded_file.read(), np.uint8), 1)
            if image is not None:
                col1, col2 = st.columns(2)
                with col1:
                    st.image(image, caption="Uploaded Image", use_column_width=True)
                
                probabilities, landmarks = process_and_predict(image)
                
                if probabilities is not None and landmarks is not None:
                    with col2:
                        st.subheader("Top 5 Predictions:")
                        top_indices = np.argsort(probabilities)[::-1][:5]
                        for i in top_indices:
                            st.write(f"{model.classes_[i]}: {probabilities[i]:.2f}")
                    
                    fig = plot_hand_landmarks(landmarks, "Detected Hand Landmarks")
                    st.pyplot(fig)
                else:
                    st.write("No hand detected in the image.")
            else:
                st.error("Failed to load the image. The file might be corrupted.")
        except Exception as e:
            st.error(f"An error occurred while processing the image: {str(e)}")

with tab2:
    st.header("View Hand Landmarks")
    
    selected_alphabets = st.multiselect("Select alphabets to view landmarks:", list(working_alphabets))

    if selected_alphabets:
        cols = st.columns(4)  # 4 columns for smaller images
        for idx, alphabet in enumerate(selected_alphabets):
            with cols[idx % 4]:
                image_path = os.path.join('asl test set', f'{alphabet.lower()}.jpeg')
                if os.path.exists(image_path):
                    try:
                        image = cv2.imread(image_path)
                        if image is not None:
                            probabilities, landmarks = process_and_predict(image)
                            if landmarks is not None:
                                fig = plot_hand_landmarks(landmarks, f"Hand Landmarks for {alphabet}")
                                st.pyplot(fig)
                            else:
                                st.error(f"No hand detected for {alphabet}")
                        else:
                            st.error(f"Failed to load image for {alphabet}")
                    except Exception as e:
                        st.error(f"Error processing image for {alphabet}")
                else:
                    st.error(f"Image not found for {alphabet}")