refactor: humaneval parser

llmdataparser/humaneval_parser.py

@@ -1,7 +1,12 @@
 from dataclasses import dataclass
 from typing import Any, ClassVar
 
-from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
 from llmdataparser.prompts import HUMANEVAL_SYSTEM_PROMPT
 
 
@@ -75,9 +80,72 @@ class HumanEvalDatasetParser(HuggingFaceDatasetParser[HumanEvalParseEntry]):
             task_name=task,  # Guarantee non-None
         )
 
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the HumanEval dataset."""
+        return DatasetDescription.create(
+            name="HumanEval",
+            purpose="Evaluate code generation capabilities through Python programming tasks",
+            source="OpenAI",
+            language="Python",
+            format="Function signatures with docstrings and unit tests",
+            characteristics=(
+                "Collection of 164 hand-written Python programming problems. Each problem "
+                "includes a function signature, docstring, example test cases, and hidden unit "
+                "tests. Problems test basic programming, algorithms, and data structure skills"
+            ),
+            citation="""@article{chen2021codex,
+                title={Evaluating Large Language Models Trained on Code},
+                author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba},
+                year={2021},
+                eprint={2107.03374},
+                archivePrefix={arXiv},
+                primaryClass={cs.LG}
+            }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for HumanEval."""
+        return [
+            EvaluationMetric.create(
+                name="pass@k",
+                type="code",
+                description="Probability that correct solution appears at least once in k samples",
+                implementation="custom_pass_at_k",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="test_success_rate",
+                type="code",
+                description="Percentage of test cases passed by the generated solution",
+                implementation="custom_test_executor",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="type_correctness",
+                type="code",
+                description="Verification of type hints and type safety in generated code",
+                implementation="custom_type_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="code_style",
+                type="code",
+                description="Compliance with Python best practices and PEP 8 guidelines",
+                implementation="custom_style_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="runtime_efficiency",
+                type="code",
+                description="Analysis of time and space complexity of the solution",
+                implementation="custom_complexity_analyzer",
+                primary=False,
+            ),
+        ]
+
 
 class HumanEvalDatasetPlusParser(HumanEvalDatasetParser):
-    """Parser for the HumanEval dataset."""
+    """Parser for the enhanced HumanEval Plus dataset with 80x more comprehensive test coverage."""
 
     _data_source: ClassVar[str] = "evalplus/humanevalplus"
     _default_task: ClassVar[str] = "default"
@@ -110,6 +178,70 @@ class HumanEvalDatasetPlusParser(HumanEvalDatasetParser):
             task_name=task,  # task is guaranteed to be str from _get_current_task
         )
 
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the HumanEval Plus dataset."""
+        return DatasetDescription.create(
+            name="HumanEval Plus",
+            purpose="Enhanced evaluation of code generation with 80x more test coverage",
+            source="EvalPlus",
+            language="Python",
+            format="Function signatures with docstrings and comprehensive test suites",
+            characteristics=(
+                "Significantly enhanced version of HumanEval with 80x more test cases. "
+                "Includes extensive edge cases, boundary conditions, stress tests, and "
+                "error handling scenarios to rigorously evaluate code correctness and robustness. "
+                "Each problem has been augmented with comprehensive testing to catch subtle bugs "
+                "and ensure production-quality code generation."
+            ),
+            citation="""@inproceedings{evalplus,
+                title = {Is Your Code Generated by Chat{GPT} Really Correct? Rigorous Evaluation of Large Language Models for Code Generation},
+                author = {Liu, Jiawei and Xia, Chunqiu Steven and Wang, Yuyao and Zhang, Lingming},
+                booktitle = {Thirty-seventh Conference on Neural Information Processing Systems},
+                year = {2023},
+                url = {https://openreview.net/forum?id=1qvx610Cu7},
+            }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for HumanEval Plus."""
+        return [
+            EvaluationMetric.create(
+                name="pass@k",
+                type="code",
+                description="Probability that correct solution appears at least once in k samples",
+                implementation="custom_pass_at_k",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="test_coverage",
+                type="code",
+                description="Percentage of edge cases and stress tests passed by the solution",
+                implementation="custom_coverage_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="error_handling",
+                type="code",
+                description="Assessment of solution's robustness in handling invalid inputs and edge cases",
+                implementation="custom_error_handler",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="performance_stress",
+                type="code",
+                description="Evaluation of solution performance under high load and stress conditions",
+                implementation="custom_stress_tester",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="code_quality",
+                type="code",
+                description="Analysis of code readability, maintainability and adherence to Python best practices",
+                implementation="custom_quality_checker",
+                primary=False,
+            ),
+        ]
+
 
 if __name__ == "__main__":
     # Example usage

tests/test_humaneval_parser.py

@@ -171,3 +171,35 @@ def test_plus_get_current_task(plus_parser, plus_sample_entry):
     """Test _get_current_task method for HumanEvalDatasetPlusParser"""
     task = plus_parser._get_current_task(plus_sample_entry)
     assert task == plus_parser._default_task
+
+
+def test_get_dataset_description(parser, plus_parser):
+    """Test dataset description generation for both parsers."""
+    # Test original HumanEval description
+    description = parser.get_dataset_description()
+    assert description.name == "HumanEval"
+    assert "code generation" in description.purpose
+    assert description.language == "Python"
+    assert "Function signatures with docstrings" in description.format
+    assert "164 hand-written Python programming problems" in description.characteristics
+    assert "chen2021codex" in description.citation
+
+    # Test HumanEval Plus description
+    plus_description = plus_parser.get_dataset_description()
+    assert plus_description.name == "HumanEval Plus"
+    assert "80x more test coverage" in plus_description.purpose
+    assert "comprehensive test suites" in plus_description.format
+    assert "edge cases" in plus_description.characteristics
+    assert "evalplus" in plus_description.citation
+
+
+def test_get_evaluation_metrics(parser, plus_parser):
+    """Test evaluation metrics generation for both parsers."""
+    # Test original HumanEval metrics
+    metrics = parser.get_evaluation_metrics()
+    assert len(metrics) == 5  # Base metrics + 2 specific metrics
+
+    # Check primary metrics - update to match actual implementation
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 1  # pass@k
+    assert any(m.name == "pass@k" for m in primary_metrics)