Add application file

app.py

@@ -0,0 +1,215 @@
+import cohere
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from transformers import pipeline
+from typing import List, Tuple
+import os
+from dotenv import load_dotenv
+import logging
+import json
+import gradio as gr
+import pandas as pd
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+load_dotenv()  # This loads the variables from .env
+
+# Initialize Cohere client, SentenceTransformer model, and QA pipeline
+co = cohere.Client(api_key = os.environ.get("COHERE_API_KEY"))
+sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
+qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
+
+def generate_questions(context: str, answer: str) -> List[str]:
+    try:
+        response = co.chat(
+            model="command-r",
+            message=f"Based on this context: '{context}' and answer: '{answer}', generate 5 diverse questions which when asked to the context returns the answer.",
+            response_format={
+                "type": "json_object",
+                "schema": {
+                    "type": "object",
+                    "required": ["question1", "question2", "question3", "question4", "question5"],
+                    "properties": {
+                        "question1": {"type": "string"},
+                        "question2": {"type": "string"},
+                        "question3": {"type": "string"},
+                        "question4": {"type": "string"},
+                        "question5": {"type": "string"}
+                    }
+                }
+            }
+        )
+        
+        json_response = response.text
+        logger.info(f"Raw JSON response: {json_response}")
+        
+        parsed_response = json.loads(json_response)
+        questions = [parsed_response[f"question{i}"] for i in range(1, 6)]
+        return questions
+    except Exception as e:
+        logger.error(f"Error in generate_questions: {e}")
+        return [f"Failed to generate question {i}" for i in range(1, 6)]
+
+def calculate_structural_diversity(questions: List[str]) -> List[float]:
+    def get_question_type(q):
+        q = q.lower()
+        if q.startswith('what'): return 1
+        elif q.startswith('why'): return 2
+        elif q.startswith('how'): return 3
+        elif q.startswith('when'): return 4
+        elif q.startswith('where'): return 5
+        else: return 0
+
+    lengths = [len(q.split()) for q in questions]
+    types = [get_question_type(q) for q in questions]
+    
+    length_scores = [1 - (abs(l - np.mean(lengths)) / np.max(lengths)) for l in lengths]
+    type_scores = [len(set(types)) / len(types) for _ in types]
+    
+    return [(l + t) / 2 for l, t in zip(length_scores, type_scores)]
+
+def calculate_semantic_relevance(context: str, answer: str, questions: List[str]) -> List[float]:
+    context_embedding = sentence_model.encode(context + " " + answer)
+    question_embeddings = sentence_model.encode(questions)
+    
+    similarities = [np.dot(context_embedding, q_emb) / (np.linalg.norm(context_embedding) * np.linalg.norm(q_emb)) 
+                    for q_emb in question_embeddings]
+    
+    return [(sim + 1) / 2 for sim in similarities]  # Normalize to 0-1 range
+
+def check_answer_precision(context: str, questions: List[str], original_answer: str) -> Tuple[List[float], List[str]]:
+    precision_scores = []
+    generated_answers = []
+    for question in questions:
+        result = qa_pipeline(question=question, context=context)
+        generated_answer = result['answer']
+        generated_answers.append(generated_answer)
+        answer_embedding = sentence_model.encode(original_answer)
+        generated_embedding = sentence_model.encode(generated_answer)
+        similarity = np.dot(answer_embedding, generated_embedding) / (np.linalg.norm(answer_embedding) * np.linalg.norm(generated_embedding))
+        precision_scores.append((similarity + 1) / 2)  # Normalize to 0-1 range
+    return precision_scores, generated_answers
+
+def calculate_composite_scores(sd_scores: List[float], sr_scores: List[float], ap_scores: List[float]) -> List[float]:
+    return [0.2 * sd + 0.4 * sr + 0.4 * ap for sd, sr, ap in zip(sd_scores, sr_scores, ap_scores)]
+
+def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame, List[pd.DataFrame], str]:
+    questions = generate_questions(context, answer)
+    
+    sd_scores = calculate_structural_diversity(questions)
+    sr_scores = calculate_semantic_relevance(context, answer, questions)
+    ap_scores, generated_answers = check_answer_precision(context, questions, answer)
+    
+    composite_scores = calculate_composite_scores(sd_scores, sr_scores, ap_scores)
+    
+    # Create detailed scores dataframe
+    detailed_scores = pd.DataFrame({
+        'Question': questions,
+        'Composite Score': composite_scores,
+        'Structural Diversity': sd_scores,
+        'Semantic Relevance': sr_scores,
+        'Answer Precision': ap_scores,
+        'Generated Answer': generated_answers
+    })
+    detailed_scores = detailed_scores.sort_values('Composite Score', ascending=False).reset_index(drop=True)
+    
+    # Create separate ranking dataframes for each metric
+    metrics = ['Composite Score', 'Structural Diversity', 'Semantic Relevance', 'Answer Precision']
+    rankings = []
+    
+    for metric in metrics:
+        df = pd.DataFrame({
+            'Rank': range(1, 6),
+            'Question': [questions[i] for i in np.argsort(detailed_scores[metric])[::-1]],
+            f'{metric}': sorted(detailed_scores[metric], reverse=True)
+        })
+        rankings.append(df)
+    
+    best_question = detailed_scores.iloc[0]['Question']
+    
+    return detailed_scores, rankings, best_question
+
+# Define sample inputs
+samples = [
+    {
+        "context": "Albert Einstein is an Austrian scientist, who has completed his higher education in ETH Zurich in Zurich, Switzerland. He was later a faculty at Princeton University.",
+        "answer": "Switzerland"
+    },
+    {
+        "context": "The Eiffel Tower, located in Paris, France, is one of the most famous landmarks in the world. It was constructed in 1889 as the entrance arch to the 1889 World's Fair. The tower is 324 meters (1,063 ft) tall and is the tallest structure in Paris.",
+        "answer": "Paris"
+    },
+    {
+        "context": "The Great Wall of China is a series of fortifications and walls built across the historical northern borders of ancient Chinese states and Imperial China to protect against nomadic invasions. It is the largest man-made structure in the world, with a total length of more than 13,000 miles (21,000 kilometers).",
+        "answer": "China"
+    }
+]
+
+def gradio_interface(context: str, answer: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, str]:
+    detailed_scores, rankings, best_question = rank_questions_with_details(context, answer)
+    return (
+        detailed_scores,
+        rankings[0],  # Composite Score Ranking
+        rankings[1],  # Structural Diversity Ranking
+        rankings[2],  # Semantic Relevance Ranking
+        rankings[3],  # Answer Precision Ranking
+        f"Best Question: {best_question}"
+    )
+
+def use_sample(sample_index: int) -> Tuple[str, str]:
+    return samples[sample_index]["context"], samples[sample_index]["answer"]
+
+# Create Gradio interface with improved layout and sample buttons
+with gr.Blocks(theme=gr.themes.Default()) as iface:
+    gr.Markdown("# Question Generator and Ranker")
+    gr.Markdown("Enter a context and an answer to generate and rank questions, or use one of the sample inputs.")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            context_input = gr.Textbox(lines=5, label="Context")
+            answer_input = gr.Textbox(lines=2, label="Answer")
+            submit_button = gr.Button("Generate Questions")
+            
+            with gr.Row():
+                sample_buttons = [gr.Button(f"Sample {i+1}") for i in range(3)]
+        
+        with gr.Column(scale=2):
+            best_question_output = gr.Textbox(label="Best Question")
+            detailed_scores_output = gr.DataFrame(label="Detailed Scores")
+    
+    with gr.Row():
+        with gr.Column():
+            composite_ranking_output = gr.DataFrame(label="Composite Score Ranking")
+        with gr.Column():
+            structural_diversity_ranking_output = gr.DataFrame(label="Structural Diversity Ranking")
+    
+    with gr.Row():
+        with gr.Column():
+            semantic_relevance_ranking_output = gr.DataFrame(label="Semantic Relevance Ranking")
+        with gr.Column():
+            answer_precision_ranking_output = gr.DataFrame(label="Answer Precision Ranking")
+
+    submit_button.click(
+        fn=gradio_interface,
+        inputs=[context_input, answer_input],
+        outputs=[
+            detailed_scores_output,
+            composite_ranking_output,
+            structural_diversity_ranking_output,
+            semantic_relevance_ranking_output,
+            answer_precision_ranking_output,
+            best_question_output
+        ]
+    )
+
+    # Set up sample button functionality
+    for i, button in enumerate(sample_buttons):
+        button.click(
+            fn=lambda i=i: use_sample(i),
+            outputs=[context_input, answer_input]
+        )
+
+
+iface.launch()

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+cohere
+numpy
+sentence-transformers
+transformers
+python-dotenv
+pandas