Spaces:
Runtime error
Runtime error
File size: 4,782 Bytes
c32baa5 6a9e2ff c32baa5 6a9e2ff 48c5ef6 c32baa5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This metric calculates both Token Overlap and Span Agreement precision, recall and f1 scores."""
import datasets
import evaluate
_CITATION = """\
@inproceedings{morante-blanco-2012-sem,
title = "*{SEM} 2012 Shared Task: Resolving the Scope and Focus of Negation",
author = "Morante, Roser and Blanco, Eduardo",
booktitle = "*{SEM} 2012: The First Joint Conference on Lexical and Computational Semantics {--} Volume 1: Proceedings of the main conference and the shared task, and Volume 2: Proceedings of the Sixth International Workshop on Semantic Evaluation ({S}em{E}val 2012)",
month = "7-8 " # jun,
year = "2012",
address = "Montr{\'e}al, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/S12-1035",
pages = "265--274",
}
"""
# TODO: Add description of the metric here
_DESCRIPTION = """\
This metric calculates both Token Overlap and Span Agreement precision, recall and f1 scores. This script is adapted from seqeval.
"""
# TODO: Add description of the arguments of the metric here
_KWARGS_DESCRIPTION = """
Calculates how good are predictions given some references, using certain scores
Args:
predictions: List of List of predicted labels.
references: List of List of reference labels.
Returns:
'token_precision': precision,
'token_recall': recall,
'token_f1': F1 score for token overlap
'span_precision': precision,
'span_recall': recall,
'span_f1': F1 score for span agreement
"""
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class SpanAgree(evaluate.Metric):
"""Calculates span agreement metric."""
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
'predictions': datasets.Sequence(datasets.Value("int8", id="label"), id="sequence"),
"references": datasets.Sequence(datasets.Value("int8", id="label"), id="sequence"),
}),
homepage="https://github.com/dannashao",
codebase_urls=["https://github.com/dannashao"],
reference_urls=["https://github.com/dannashao"]
)
def _compute(self, predictions, references):
"""Returns the scores"""
# TOKEN LEVEL
tn, fp, fn, tp = 0,0,0,0
for span_true, span_pred in zip(references, predictions):
for token_true, token_pred in zip(span_true, span_pred):
if token_true == 1:
if token_pred == 1:
tp += 1
else:
fn += 1
else:
if token_pred == 1:
fp += 1
else:
tn += 1
precision = tp / (tp + fp) if tp + fp > 0 else 0
recall = tp / (tp + fn) if tp + fn > 0 else 0
f1 = 2 * (precision * recall) / (precision + recall) if precision + recall > 0 else 0
# SPAN LEVEL
tn, fp, fn, tp = 0,0,0,0
for span_true, span_pred in zip(references, predictions):
if 1 in span_true:
if span_true == span_pred:
tp += 1
elif all([(yt == 0 or (yt == 1 and predictions[i] == 1)) for i, yt in enumerate(references)]):
fp += 1
else:
fp += 1
fn += 1
else:
if 1 in span_pred:
fp += 1
fn += 1
else:
tn += 1
span_precision = tp / (tp + fp) if tp + fp > 0 else 0
span_recall = tp / (tp + fn) if tp + fn > 0 else 0
span_f1 = 2 * (span_precision * span_recall) / (span_precision + span_recall) if span_precision + span_recall > 0 else 0
scores = {"token_precision":precision, "token_recall":recall, "token_f1":f1,
"span_precision":span_precision, "span_recall":span_recall, "span_f1":span_f1}
return scores
|