Upload folder using huggingface_hub

generate_training_data.py

@@ -0,0 +1,319 @@
+import httpx
+import json
+import random
+from typing import List, Dict
+import threading
+import queue
+import time
+import logging
+import re
+import os
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+import httpx
+import json
+import random
+from typing import List, Dict
+import threading
+import queue
+import time
+import logging
+import re
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Base URL for the API
+BASE_URL = "http://localhost:6002/v1/chat/completions"
+
+# API key (replace with your actual key)
+API_KEY = "your-api-key-here"
+
+# Existing MESSAGES and example_jsonl (abbreviated for brevity)
+MESSAGES = """
+### Task: Generate high quality A Graduate-Level Google-Proof Q&A Benchmark expert detailed multiple choice questions and answers.
+Then list the correct answer and 3 incorrect answers.  Provide a detailed explanation for the correct answer only.
+
+Here's a sophisticated example sample data output would look like: 
+
+-------
+### Category: cosmology
+In cosmological studies, various observational techniques are employed to constrain the equation of state of dark energy. Consider the following methods:
+
+Intensity mapping of neutral hydrogen emission lines at frequencies between 600 MHz and 1.4 GHz
+Intensity mapping of CO emission lines at frequencies between 10 GHz to 20 GHz
+Galaxy-redshift surveys at redshift < 2
+Measurements of frequency shifts in absorption lines of cold damped Lyman-alpha systems at redshift < 2
+
+Which of these methods is least effective in directly constraining the dark energy equation of state, and why?
+
+
+A) Intensity mapping of neutral hydrogen emission lines at frequencies between 600 MHz and 1.4 GHz (incorrect)
+B) Intensity mapping of CO emission lines at frequencies between 10 GHz to 20 GHz (incorrect)
+C) Galaxy-redshift surveys at redshift < 2 (incorrect)
+D) Measurements of frequency shifts in absorption lines of cold damped Lyman-alpha systems at redshift < 2 (correct)
+
+
+Answer: D) Measurements of frequency shifts in absorption lines of cold damped Lyman-alpha systems at redshift < 2
+
+The method least effective in directly constraining the dark energy equation of state is:
+Measurements of frequency shifts in absorption lines of cold damped Lyman-alpha systems at redshift < 2
+
+Explanation: While this method can provide valuable information about the expansion of the universe, it is less directly applicable to constraining the dark energy equation of state compared to the other options. The primary reasons are:
+
+Limited redshift range: The specified redshift range (z < 2) is relatively low, whereas dark energy effects become more prominent at higher redshifts.
+Indirect measurement: This method primarily measures the expansion rate of the universe rather than directly probing dark energy properties.
+Systematics and uncertainties: Lyman-alpha systems can be affected by various astrophysical processes, introducing complexities in interpreting the data specifically for dark energy constraints.
+Lack of large-scale structure information: Unlike galaxy surveys or intensity mapping techniques, this method doesn't provide information about the growth of large-scale structure, which is crucial for distinguishing between different dark energy models.
+
+The other methods listed (intensity mapping and galaxy-redshift surveys) are generally more effective for constraining the dark energy equation of state because they provide complementary information about both the expansion history and the growth of structure in the universe, allowing for better constraints on dark energy properties. Also make a correct and 3 incorrect answers for Q and A.
+-------
+
+Return a new sophisticated example based on reference format, make sure yours is unique and data expert-level question provided with great lengthy explanation:
+
+Do not talk about the answers in the question in the explanation. Only the reasoning behind the correct answer should be explained. 
+
+### Category: {category}"""
+
+
+example_jsonl = """{"category": "Biology", "question": "Given the observed correlation between the mutation in protein X and accelerated muscle growth, what is the most plausible mechanism by which this mutation could lead to the observed phenotype?", "correct_answer": "Increased protein X activity", "incorrect_answer1": "Decreased protein X activity", "incorrect_answer2": "Direct interaction with muscle fibers", "incorrect_answer3": "Alteration of gene expression in unrelated pathways", "explanation": "The most plausible mechanism is: \n\nIncreased protein X activity: \n\nExplanation:\nThe mutation likely leads to enhanced protein X function. \n* Increased protein synthesis: The mutation could increase the rate of protein X translation, resulting in a higher number of functional protein X molecules.\n* Enhanced protein stability:  The mutation might make the protein X molecule more resistant to degradation, leading to a longer lifespan and increased overall activity.\n* Altered protein binding: The mutation could modify protein X's ability to bind to other molecules, potentially activating downstream signaling pathways involved in muscle growth.\n\nIncorrect Answers:\n1. Decreased protein X activity: This is unlikely, as the mutation leads to increased expression levels, suggesting a gain-of-function effect.\n2. Direct interaction with muscle fibers: Protein X might not directly interact with muscle cells, so this mechanism is less probable.\n3. Alteration of gene expression in unrelated pathways: While the mutation could have broader effects, the strongest link is to the increased protein X levels and its potential role in muscle growth."}"""
+
+# List of categories
+categories = [
+    "Law", 
+    # "Physics", "Mathematics", "Computer Science", "Biology",
+    # "Chemistry", "History", "Literature", "Economics", "Psychology"
+]
+
+categories = ["Metalworking (smelting, casting, forging)", "Machine Tools (lathes, mills, drills)", "Woodworking (saws, planers, jointers)", "Textiles (spinning, weaving, knitting)", "Ceramics and Glassmaking", "Paper Production", "Chemical Manufacturing (acids, bases, solvents)", "Plastics and Polymers", "Electrical Systems (generators, motors, transformers)", "Electronics (basic components, circuit boards)", "Fuel Production (biofuels, alcohol distillation)", "Refrigeration and Air Conditioning", "Food Processing and Preservation", "Agricultural Machinery", "Transportation (bicycles, carts, simple engines)", "Mining and Mineral Extraction", "Water Treatment and Distribution", "Waste Management and Recycling", "Construction Materials (cement, concrete, bricks)", "Energy Production (solar panels, wind turbines, hydropower)", "Toolmaking (hand tools, precision instruments)", "Rope and Cordage Manufacturing", "Adhesives and Sealants", "Paint and Coatings", "Lubricants and Greases", "Batteries and Energy Storage", "Welding and Joining Techniques", "Pneumatics and Hydraulics", "Pumps and Compressors", "Gears and Power Transmission", "Bearings and Bushings", "Springs and Shock Absorbers", "Fasteners (screws, bolts, rivets)", "Seals and Gaskets", "Filters and Separation Technology", "Heating Systems (furnaces, boilers)", "Plumbing and Pipe Fitting", "Insulation Materials", "Lighting Technology", "Timekeeping Devices", "Measurement and Surveying Tools", "Communication Systems (radio, telegraph)", "Printing and Information Dissemination", "Optics and Lens Crafting", "Medical Equipment Manufacturing", "Pharmaceutical Production", "Fertilizer and Soil Amendments", "Pest Control Methods", "Animal Husbandry Equipment", "Tanning and Leather Working", "Brewing and Fermentation", "Distillation Equipment", "Fiber Processing (cotton gins, wool carders)", "Metallurgy and Alloying", "Foundry Techniques", "Sheet Metal Working", "Blacksmithing and Ironworking", "Precision Grinding", "Heat Treatment of Metals", "Woodturning and Carving", "Joinery and Cabinetmaking", "Masonry and Stoneworking", "Roofing and Weatherproofing", "Paint and Pigment Production", "Dyeing and Coloring Techniques", "Pottery and Ceramics Forming", "Kiln Design and Operation", "Glass Blowing and Forming", "Fiber Reinforced Composites", "Rubber Processing and Vulcanization", "Oil Pressing and Refining", "Soap and Detergent Manufacturing", "Candle Making", "Rope Making and Cordage", "Basketry and Weaving", "Papermaking and Pulp Processing", "Bookbinding and Preservation", "Ink and Writing Implement Production", "Clockmaking and Timekeeping", "Lock and Key Manufacturing", "Simple Computing Devices", "Analog Instrumentation", "Geolocation and Mapping Tools", "Water Wheel and Turbine Design", "Wind Power Mechanisms", "Solar Thermal Technologies", "Biogas Production", "Charcoal Making", "Beekeeping Equipment", "Mushroom Cultivation", "Hydroponics and Aeroponics", "Greenhouse Design and Operation", "Seed Cleaning and Storage", "Food Dehydration Equipment", "Canning and Bottling Technology", "Refrigeration Without Electricity", "Evaporative Cooling Systems", "Water Filtration and Purification", "Waste Composting Systems", "Anaerobic Digestion", "Bioremediation Techniques"];
+
+# Thread-safe queues and locks
+result_queue = queue.Queue()
+category_queue = queue.Queue()
+file_lock = threading.Lock()
+
+def create_chat_messages(category: str) -> List[Dict[str, str]]:
+    return [
+        # {"role": "system", "content": "You are an expert in creating graduate-level, Google-proof multiple-choice questions for various academic fields."},
+        {"role": "user", "content": f"""Generate a high-quality, graduate-level, Google-proof multiple-choice question for the category: {category}.
+        
+        Include:
+        1. The question
+        2. Four answer choices (A, B, C, D) with one correct answer
+        3. Indication of which answer is correct
+        4. A detailed explanation for why the correct answer is right and the others are wrong
+        
+        Make sure the question is sophisticated and challenging, suitable for graduate-level students in the field.
+        
+        Here's an example of a well-structured question:
+        {MESSAGES.format(category=category)}"""}
+    ]
+
+def generate_question(category: str) -> str:
+    messages = create_chat_messages(category)
+    
+    payload = {
+        "model": "gemma",  # or whatever model name your local server uses
+        "messages": messages,
+        "max_tokens": 2000,
+        "temperature": 0.7,
+        "top_p": 0.841,
+        "frequency_penalty": 0,
+        "presence_penalty": 0,
+        "n": 1,
+        "stream": False,
+    }
+
+    logging.info(f"Generating question for category: {category}")
+    with httpx.Client() as client:
+        response = client.post(BASE_URL, json=payload, timeout=60)
+        response.raise_for_status()
+        result = response.json()
+
+    res = result['choices'][0]['message']['content']
+    print(f"Generated question for {category}:\n\n{res}")
+    return res
+
+def convert_to_jsonl(category: str, text: str) -> dict:
+    messages = [
+        {"role": "user", "content": f"""You are an expert in parsing and structuring data into JSON format. Convert the following text into a JSON object with these keys: 
+        "category", "question", "correct_answer", "incorrect_answer1", "incorrect_answer2", "incorrect_answer3", "explanation"
+
+        example json: 
+        ---
+        {example_jsonl}
+        ---
+
+        Here's the text to convert:
+
+        {text}
+
+        Return only the JSON object, nothing else."""},
+        {"role": "assistant", "content": '```jsonl\n'},
+    ]
+    
+    client = httpx.Client(timeout=240)
+    
+    json_schema = {
+        "type": "object",
+        "properties": {
+            "category": {"type": "string"},
+            "question": {"type": "string"},
+            "correct_answer": {"type": "string"},
+            "incorrect_answer1": {"type": "string"},
+            "incorrect_answer2": {"type": "string"},
+            "incorrect_answer3": {"type": "string"},
+            "explanation": {"type": "string"}
+        },
+        "required": ["category", "question", "correct_answer", "incorrect_answer1", "incorrect_answer2", "incorrect_answer3", "explanation"]
+    }
+    
+    data = {
+        "model": "gemma",
+        "messages": messages,
+        "guided_json": json_schema,
+        "guided_decoding_backend": "lm-format-enforcer",
+        "temperature": 0.1,
+        "stop" : ["Let me know"],
+        "max_tokens": 1000
+    }
+    
+    try:
+        response = client.post(BASE_URL, json=data, headers={
+            "Authorization": f"Bearer YOUR_API_KEY_HERE",
+            "Content-Type": "application/json"
+        })
+        response.raise_for_status()
+        result = response.json()
+        parsed_json = json.loads(result['choices'][0]['message']['content'])
+        print(f"JSON parsed for {category}:\n\n{json.dumps(parsed_json, indent=4)}")
+        return parsed_json
+    except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
+        raise ValueError(f"Failed to parse JSON: {str(e)}")
+
+def process_category(category: str):
+    try:
+        logging.info(f"Processing category: {category}")
+        question_text = generate_question(category)
+        parsed_json = convert_to_jsonl(category, question_text)
+        save_to_jsonl(json.dumps(parsed_json), f"{category.lower().replace(' ', '_')}.jsonl")
+        
+        result_queue.put((category, json.dumps(parsed_json)))
+    except Exception as e:
+        logging.error(f"An error occurred while processing {category}: {str(e)}")
+
+def save_to_jsonl(jsonl_text: str, filename: str):
+    logging.info(f"Attempting to save to file: {filename}")
+    logging.info(f"Content to be saved: {jsonl_text}")
+    
+
+    try:
+        with open(filename, 'a', encoding='utf-8') as f:
+            f.write(jsonl_text + '\n')
+        logging.info(f"Successfully appended to {filename}")
+    except Exception as e:
+        logging.error(f"Error while writing to {filename}: {str(e)}")
+
+    # Verify file content after writing
+    try:
+        with open(filename, 'r', encoding='utf-8') as f:
+            updated_content = f.read()
+        logging.info(f"Updated content of {filename}:\n{updated_content}")
+    except Exception as e:
+        logging.error(f"Error while reading {filename} after write: {str(e)}")
+
+
+
+
+def clean_and_parse_json(text: str) -> Dict:
+    # First, remove the ```jsonl wrapper if present
+    text = re.sub(r'```jsonl\s*|\s*```', '', text)
+    
+    json_pattern = r'(?s)\{.*\}'
+    match = re.search(json_pattern, text)
+    if not match:
+        raise ValueError("No JSON-like structure found in the text")
+
+    json_str = match.group()
+    json_str = json_str.strip().strip('"').strip("'")
+    json_str = re.sub(r"(?<!\\)'", '"', json_str)
+    
+    import ast 
+    try:
+        # parsed_json = json.loads(json_str)
+        parsed_json = ast.literal_eval(json_str)
+        return parsed_json
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to parse JSON: {str(e)}")
+
+
+# def process_category(category: str):
+#     try:
+#         logging.info(f"Processing category: {category}")
+#         question_text = generate_question(category)
+#         jsonl_text = convert_to_jsonl(category, question_text)
+        
+#         try:
+#             parsed_json = clean_and_parse_json(jsonl_text)
+#             result_queue.put((category, json.dumps(parsed_json)))
+#         except ValueError as e:
+#             logging.error(f"Failed to parse JSON for {category}: {str(e)}")
+#             save_to_jsonl(jsonl_text, f"{category.lower().replace(' ', '_')}_failed.jsonl")
+#     except Exception as e:
+#         logging.error(f"An error occurred while processing {category}: {str(e)}")
+
+def worker():
+    while True:
+        try:
+            category = category_queue.get(timeout=1)  # Wait for 1 second
+            if category is None:
+                break
+            process_category(category)
+        except queue.Empty:
+            continue  # If queue is empty, continue the loop
+        except Exception as e:
+            logging.error(f"Worker thread encountered an error: {str(e)}")
+        finally:
+            category_queue.task_done()
+
+def main():
+    num_threads = 6  # Reduced number of threads to avoid overwhelming the API
+    output_file = "all_questions.jsonl"
+
+    threads = []
+    for _ in range(num_threads):
+        t = threading.Thread(target=worker)
+        t.daemon = True  # Set threads as daemon
+        t.start()
+        threads.append(t)
+
+    for category in categories:
+        category_queue.put(category)
+
+    # Wait for all tasks to be completed
+    category_queue.join()
+
+    # Signal threads to exit
+    for _ in range(num_threads):
+        category_queue.put(None)
+
+    # Wait for all threads to finish
+    for t in threads:
+        t.join()
+
+    while not result_queue.empty():
+        category, jsonl_text = result_queue.get()
+        save_to_jsonl(jsonl_text, output_file)
+        logging.info(f"Question for {category} saved to {output_file}")
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        logging.info("Script interrupted by user. Shutting down gracefully...")
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {str(e)}")
+    finally:
+        logging.info("Script execution completed.")