typo

harim_plus.py

@@ -7,16 +7,26 @@ from harim_scorer import Harimplus_Scorer
 
 logger = evaluate.logging.get_logger(__name__)
 
-CODEBASE_URL=''
-PAPER_URL='TBA'
+CODEBASE_URL='https://huggingface.co/spaces/NCSOFT/harim_plus'
+PAPER_URL='https://arxiv.org/abs/2211.12118'
 
 _CITATION = """\
-@inproceedings{harimplus,
-  title={HaRiM+: Evaluating Summary Quality with Hallucination Risk},
-  author={Seonil Son, Junsoo Park, Jeong-in Hwang, Hyungjong Noh, Yeonsoo Lee},
-  booktitle={AACL},
-  year={2022},
-  url={TBA}
+@inproceedings{son-etal-2022-harim,
+    title = "{H}a{R}i{M}$^+$: Evaluating Summary Quality with Hallucination Risk",
+    author = "Son, Seonil (Simon)  and
+      Park, Junsoo  and
+      Hwang, Jeong-in  and
+      Lee, Junghwa  and
+      Noh, Hyungjong  and
+      Lee, Yeonsoo",
+    booktitle = "Proceedings of the 2nd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics and the 12th International Joint Conference on Natural Language Processing",
+    month = nov,
+    year = "2022",
+    address = "Online only",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2022.aacl-main.66",
+    pages = "895--924",
+    abstract = "One of the challenges of developing a summarization model arises from the difficulty in measuring the factual inconsistency of the generated text. In this study, we reinterpret the decoder overconfidence-regularizing objective suggested in (Miao et al., 2021) as a hallucination risk measurement to better estimate the quality of generated summaries. We propose a reference-free metric, HaRiM+, which only requires an off-the-shelf summarization model to compute the hallucination risk based on token likelihoods. Deploying it requires no additional training of models or ad-hoc modules, which usually need alignment to human judgments. For summary-quality estimation, HaRiM+ records state-of-the-art correlation to human judgment on three summary-quality annotation sets: FRANK, QAGS, and SummEval. We hope that our work, which merits the use of summarization models, facilitates the progress of both automated evaluation and generation of summary.",
 }
 """
 
@@ -56,7 +66,7 @@ Examples:
     >>> scorer = evaluate.load("NCSOFT/harim_plus") #, pretrained_name='PRETRAINEDNAME', tokenizer=TOKENIZER # optional
     >>> results = scorer.compute(predictions=summaries, references=articles) # use_aggregator=True # optional
     >>> print([round(v, 2) for v in results["harim+"]])
-    [0.4, 0.4]
+    [float, float]
 """
 
 
@@ -94,7 +104,7 @@ class Harimplus(evaluate.Metric):
 
     def _download_and_prepare(self, dl_manager):
         pretrained_name = self.myconfig['pretrained_name']
-        is_custom_tokenzer = self.myconfig['tokenizer'] is not None
+        is_custom_tokenizer = self.myconfig['tokenizer'] is not None
         logger.warning(
             "Loading HaRiM+ score"
             f"\tpretrained_name = {pretrained_name}"