import streamlit as st import cv2 import numpy as np import mediapipe as mp import pickle # Load models directly with open('random_forest_model.pkl', 'rb') as file: random_forest_model = pickle.load(file) with open('svm_model.pkl', 'rb') as file: svm_model = pickle.load(file) with open('hard_voting_classifier.pkl', 'rb') as file: hard_voting_model = pickle.load(file) with open('soft_voting_classifier.pkl', 'rb') as file: soft_voting_model = pickle.load(file) label_classes = np.load('label_classes.npy', allow_pickle=True) models = { 'Random Forest': random_forest_model, 'SVM': svm_model, 'Hard Voting': hard_voting_model, 'Soft Voting': soft_voting_model } # Initialize MediaPipe Hands mp_hands = mp.solutions.hands hands = mp_hands.Hands(static_image_mode=True, max_num_hands=1, min_detection_confidence=0.5) def process_landmarks(hand_landmarks): x_ = [landmark.x for landmark in hand_landmarks.landmark] y_ = [landmark.y for landmark in hand_landmarks.landmark] min_x, min_y = min(x_), min(y_) return [coord - min_val for landmark in hand_landmarks.landmark for coord, min_val in zip((landmark.x, landmark.y), (min_x, min_y))] def get_model_predictions(data_aux, model): if hasattr(model, 'predict_proba'): probabilities = model.predict_proba([data_aux])[0] elif hasattr(model, 'decision_function'): scores = model.decision_function([data_aux])[0] probabilities = (scores - scores.min()) / (scores.max() - scores.min()) else: class_pred = model.predict([data_aux])[0] probabilities = np.zeros(len(label_classes)) probabilities[class_pred] = 1 # Add this check if not hasattr(model, 'monotonic_cst'): model.monotonic_cst = None top_index = np.argmax(probabilities) return label_classes[top_index], probabilities[top_index] def predict_alphabet(image): image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) results = hands.process(image_rgb) if not results.multi_hand_landmarks: return None, None data_aux = process_landmarks(results.multi_hand_landmarks[0]) predictions = {} for model_name, model in models.items(): prediction, probability = get_model_predictions(data_aux, model) predictions[model_name] = (prediction, probability) return predictions, results.multi_hand_landmarks[0] def draw_hand_landmarks(image, hand_landmarks): for landmark in hand_landmarks.landmark: h, w, _ = image.shape cx, cy = int(landmark.x * w), int(landmark.y * h) cv2.circle(image, (cx, cy), 5, (0, 255, 0), -1) return image # Streamlit App st.title("ASL Hand Shape Recognition App") uploaded_file = st.file_uploader("Choose a hand image...", type=["jpg", "jpeg", "png"]) if uploaded_file is not None: image = cv2.imdecode(np.frombuffer(uploaded_file.read(), np.uint8), 1) st.image(image, channels="BGR", caption="Uploaded Hand Image") predictions, hand_landmarks = predict_alphabet(image) if predictions and hand_landmarks: st.subheader("Predictions:") for model_name, (prediction, probability) in predictions.items(): st.write(f"{model_name}: {prediction} (Probability: {probability:.2f})") st.subheader("Hand Landmarks:") landmark_image = draw_hand_landmarks(image.copy(), hand_landmarks) st.image(landmark_image, channels="BGR", caption="Hand Landmarks") else: st.write("No hand detected in the image.") # User input to draw hand sign (placeholder) user_input = st.text_input("Enter alphabet to draw hand sign:") if user_input: if len(user_input) == 1 and user_input.isalpha(): st.write(f"Drawing hand sign for alphabet: {user_input.upper()}") # Placeholder for drawing hand sign st.write("(Hand sign drawing functionality not implemented)") else: st.error("Please enter a single alphabet.") # Release MediaPipe resources hands.close()