refactor: mgsm parser

llmdataparser/mgsm_parser.py

@@ -1,7 +1,12 @@
 from dataclasses import dataclass
 from typing import Any, ClassVar
 
-from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
 from llmdataparser.prompts import MGSM_SYSTEM_PROMPT
 
 
@@ -93,6 +98,86 @@ class MGSMDatasetParser(HuggingFaceDatasetParser[MGSMParseEntry]):
             language=task,
         )
 
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a description of the Multilingual Grade School Math dataset."""
+        return DatasetDescription.create(
+            name="Multilingual Grade School Math (MGSM)",
+            purpose="Evaluate multilingual chain-of-thought reasoning capabilities in mathematical problem solving",
+            source="https://huggingface.co/datasets/juletxara/mgsm",
+            language="Multilingual (11 languages)",
+            format="Word problems with numerical answers and solution steps",
+            characteristics=(
+                "Human-translated version of 250 GSM8K problems into 10 additional languages. "
+                "Each problem includes the original question from GSM8K, its translations, "
+                "numerical answer, and solution steps. The benchmark is designed to evaluate "
+                "language models' ability to perform mathematical reasoning across different languages."
+            ),
+            citation="""@misc{shi2022language,
+                        title={Language Models are Multilingual Chain-of-Thought Reasoners},
+                        author={Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and Jason Wei},
+                        year={2022},
+                        eprint={2210.03057},
+                        archivePrefix={arXiv},
+                        primaryClass={cs.CL}
+                    }
+                    @article{cobbe2021gsm8k,
+                        title={Training Verifiers to Solve Math Word Problems},
+                        author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},
+                        journal={arXiv preprint arXiv:2110.14168},
+                        year={2021}
+                    }""",
+            additional_info={
+                "languages": [
+                    "Bengali",
+                    "German",
+                    "English",
+                    "Spanish",
+                    "French",
+                    "Japanese",
+                    "Russian",
+                    "Swahili",
+                    "Telugu",
+                    "Thai",
+                    "Chinese",
+                ],
+                "size": "250 problems translated into each language",
+                "base_dataset": "GSM8K (Grade School Math 8K)",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MGSM dataset."""
+        return [
+            EvaluationMetric.create(
+                name="exact_match",
+                type="string",
+                description="Exact match comparison between predicted and correct numerical answers",
+                implementation="custom_exact_match",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="solution_validity",
+                type="text",
+                description="Assessment of whether the solution steps are mathematically valid and complete",
+                implementation="custom_solution_validator",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="step_accuracy",
+                type="numerical",
+                description="Accuracy of intermediate calculation steps (e.g., <<48/2=24>>)",
+                implementation="custom_step_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="cross_lingual_consistency",
+                type="comparison",
+                description="Consistency of model performance across different language versions of the same problem",
+                implementation="custom_language_comparator",
+                primary=False,
+            ),
+        ]
+
 
 if __name__ == "__main__":
     from pprint import pprint

tests/test_mgsm_parser.py

@@ -182,3 +182,89 @@ def test_system_prompt_override(mgsm_parser):
 
     entry = parser.process_entry(test_entry, task_name="en")
     assert custom_prompt in entry.prompt
+
+
+def test_get_dataset_description(mgsm_parser):
+    """Test dataset description generation."""
+    description = mgsm_parser.get_dataset_description()
+
+    assert description.name == "Multilingual Grade School Math (MGSM)"
+    assert "multilingual chain-of-thought reasoning" in description.purpose.lower()
+    assert "juletxara/mgsm" in description.source
+    assert description.language == "Multilingual (11 languages)"
+    assert "word problems" in description.format.lower()
+    assert "numerical answers" in description.format.lower()
+    assert "solution steps" in description.format.lower()
+
+    # Check characteristics
+    assert "250" in description.characteristics
+    assert "gsm8k" in description.characteristics.lower()
+    assert "translations" in description.characteristics.lower()
+    assert "mathematical reasoning" in description.characteristics.lower()
+
+    # Check citations
+    assert "shi2022language" in description.citation
+    assert "cobbe2021gsm8k" in description.citation
+    assert (
+        "Language Models are Multilingual Chain-of-Thought Reasoners"
+        in description.citation
+    )
+    assert "Training Verifiers to Solve Math Word Problems" in description.citation
+
+    # Check additional info
+    assert description.additional_info is not None
+    assert len(description.additional_info["languages"]) == 11
+    assert "English" in description.additional_info["languages"]
+    assert "Chinese" in description.additional_info["languages"]
+    assert (
+        description.additional_info["size"]
+        == "250 problems translated into each language"
+    )
+    assert description.additional_info["base_dataset"] == "GSM8K (Grade School Math 8K)"
+
+
+def test_get_evaluation_metrics(mgsm_parser):
+    """Test evaluation metrics generation."""
+    metrics = mgsm_parser.get_evaluation_metrics()
+
+    # Check total number of metrics
+    assert len(metrics) == 4
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 3
+
+    # Verify specific metrics exist with correct properties
+    metric_names = {m.name for m in metrics}
+    assert "exact_match" in metric_names
+    assert "solution_validity" in metric_names
+    assert "step_accuracy" in metric_names
+    assert "cross_lingual_consistency" in metric_names
+
+    # Check specific metric properties
+    exact_match_metric = next(m for m in metrics if m.name == "exact_match")
+    assert exact_match_metric.type == "string"
+    assert exact_match_metric.primary is True
+    assert "numerical answers" in exact_match_metric.description.lower()
+    assert "custom_exact_match" in exact_match_metric.implementation
+
+    solution_metric = next(m for m in metrics if m.name == "solution_validity")
+    assert solution_metric.type == "text"
+    assert solution_metric.primary is True
+    assert "mathematically valid" in solution_metric.description.lower()
+    assert "custom_solution_validator" in solution_metric.implementation
+
+    step_metric = next(m for m in metrics if m.name == "step_accuracy")
+    assert step_metric.type == "numerical"
+    assert step_metric.primary is True
+    assert "calculation steps" in step_metric.description.lower()
+    assert "custom_step_accuracy" in step_metric.implementation
+
+    # Check cross-lingual metric specifically
+    cross_lingual_metric = next(
+        m for m in metrics if m.name == "cross_lingual_consistency"
+    )
+    assert cross_lingual_metric.type == "comparison"
+    assert cross_lingual_metric.primary is False
+    assert "different language versions" in cross_lingual_metric.description.lower()
+    assert "custom_language_comparator" in cross_lingual_metric.implementation