|
import evaluate |
|
import pytest |
|
|
|
ner_eval = evaluate.load("ner_eval.py") |
|
|
|
test_cases = [ |
|
{ |
|
"predictions": ["B-PER", "I-PER", "O", "B-LOC", "I-LOC", "O", "O", "B-ORG"], |
|
"references": ["B-PER", "I-PER", "O", "B-LOC", "I-LOC", "O", "O", "B-ORG"], |
|
"results": { |
|
"overall": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
"LOC": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
"PER": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
"ORG": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
}, |
|
}, |
|
{ |
|
"predictions": [ |
|
"B-LOC", |
|
"I-LOC", |
|
"O", |
|
"B-PER", |
|
"I-PER", |
|
"I-PER", |
|
"I-PER", |
|
"O", |
|
"B-LOC", |
|
"O", |
|
], |
|
"references": [ |
|
"B-LOC", |
|
"I-LOC", |
|
"O", |
|
"B-PER", |
|
"I-PER", |
|
"I-PER", |
|
"I-PER", |
|
"O", |
|
"B-LOC", |
|
"O", |
|
], |
|
"results": { |
|
"overall": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
"LOC": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
"PER": { |
|
"strict_precision": 1.0, |
|
"strict_recall": 1.0, |
|
"strict_f1": 1.0, |
|
"ent_type_precision": 1.0, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 1.0, |
|
"partial_precision": 1.0, |
|
"partial_recall": 1.0, |
|
"partial_f1": 1.0, |
|
"exact_precision": 1.0, |
|
"exact_recall": 1.0, |
|
"exact_f1": 1.0, |
|
}, |
|
}, |
|
}, |
|
{ |
|
"predictions": ["O", "B-LOC", "I-LOC", "B-PER", "I-PER", "O", "B-ORG"], |
|
"references": ["O", "B-LOC", "I-LOC", "O", "B-PER", "I-PER", "O", "B-ORG"], |
|
}, |
|
{ |
|
"predictions": ["B-PER", "O", "B-LOC", "I-LOC", "O", "B-ORG", "I-ORG"], |
|
"references": ["B-PER", "I-PER", "O", "B-LOC", "I-LOC", "O", "B-ORG"], |
|
"results": { |
|
"overall": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.0, |
|
"ent_type_recall": 0.0, |
|
"ent_type_f1": 0, |
|
"partial_precision": 0.0, |
|
"partial_recall": 0.0, |
|
"partial_f1": 0, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"ORG": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.0, |
|
"ent_type_recall": 0.0, |
|
"ent_type_f1": 0, |
|
"partial_precision": 0.0, |
|
"partial_recall": 0.0, |
|
"partial_f1": 0, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"PER": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.0, |
|
"ent_type_recall": 0.0, |
|
"ent_type_f1": 0, |
|
"partial_precision": 0.0, |
|
"partial_recall": 0.0, |
|
"partial_f1": 0, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"LOC": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.0, |
|
"ent_type_recall": 0.0, |
|
"ent_type_f1": 0, |
|
"partial_precision": 0.0, |
|
"partial_recall": 0.0, |
|
"partial_f1": 0, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
}, |
|
}, |
|
{ |
|
"predictions": [ |
|
"B-LOC", |
|
"I-LOC", |
|
"I-LOC", |
|
"B-ORG", |
|
"I-ORG", |
|
"O", |
|
"B-PER", |
|
"I-PER", |
|
"I-PER", |
|
"O", |
|
], |
|
"references": [ |
|
"B-LOC", |
|
"I-LOC", |
|
"O", |
|
"O", |
|
"B-ORG", |
|
"I-ORG", |
|
"O", |
|
"B-PER", |
|
"I-PER", |
|
"O", |
|
], |
|
"results": { |
|
"overall": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 2 / 3, |
|
"ent_type_recall": 2 / 3, |
|
"ent_type_f1": 2 / 3, |
|
"partial_precision": 1 / 3, |
|
"partial_recall": 1 / 3, |
|
"partial_f1": 1 / 3, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"ORG": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.0, |
|
"ent_type_recall": 0.0, |
|
"ent_type_f1": 0, |
|
"partial_precision": 0.0, |
|
"partial_recall": 0.0, |
|
"partial_f1": 0, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"PER": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.5, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 2 / 3, |
|
"partial_precision": 0.25, |
|
"partial_recall": 0.5, |
|
"partial_f1": 1 / 3, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
"LOC": { |
|
"strict_precision": 0.0, |
|
"strict_recall": 0.0, |
|
"strict_f1": 0, |
|
"ent_type_precision": 0.5, |
|
"ent_type_recall": 1.0, |
|
"ent_type_f1": 2 / 3, |
|
"partial_precision": 0.25, |
|
"partial_recall": 0.5, |
|
"partial_f1": 1 / 3, |
|
"exact_precision": 0.0, |
|
"exact_recall": 0.0, |
|
"exact_f1": 0, |
|
}, |
|
}, |
|
}, |
|
] |
|
|
|
|
|
def compare_results(result1, result2): |
|
|
|
if isinstance(result1, dict): |
|
for key in result1.keys(): |
|
if not compare_results(result1[key], result2[key]): |
|
return False |
|
return True |
|
elif isinstance(result1, list): |
|
for item1, item2 in zip(result1, result2): |
|
if not compare_results(item1, item2): |
|
return False |
|
return True |
|
else: |
|
return result1 == result2 |
|
|
|
|
|
@pytest.mark.parametrize("case", test_cases) |
|
def test_metric(case): |
|
if "results" not in case: |
|
with pytest.raises(ValueError): |
|
results = ner_eval.compute( |
|
predictions=[case["predictions"]], references=[case["references"]] |
|
) |
|
else: |
|
results = ner_eval.compute( |
|
predictions=[case["predictions"]], references=[case["references"]] |
|
) |
|
assert compare_results(results, case["results"]) |
|
|
