File size: 3,932 Bytes
345ee20 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
__author__ = 'tylin'
from .tokenizer.ptbtokenizer import PTBTokenizer
from .spice.spice import Spice
class SpiceEval():
def __init__(self):
self.evalImgs = []
self.eval = {}
self.imgToEval = {}
self.spice = Spice()
self.tokenizer = PTBTokenizer()
"""
The input have structure
{'123': [{'image_id':123, 'caption': 'xxxxx'}, {'image_id':123, 'caption': 'yyy'}], ...}
"""
def evaluate(self, gts, res):
assert set(gts.keys()) == set(res.keys())
imgIds = list(gts.keys())
gts = self.tokenizer.tokenize(gts)
res = self.tokenizer.tokenize(res)
# =================================================
# Set up scorers
# =================================================
# =================================================
# Compute scores
# =================================================
print('computing %s score...'%(self.spice.method()))
score, scores = self.spice.compute_score(gts, res)
print("%s: %0.3f"%("spice", score))
self.eval['spice'] = score
print(scores)
for imgId, score in zip(sorted(imgIds), scores):
if not imgId in self.imgToEval:
self.imgToEval[imgId] = {}
self.imgToEval[imgId]["image_id"] = imgId
self.imgToEval[imgId]["spice"] = score
return self.eval['spice'], self.imgToEval
class COCOEvalCapSpice:
def __init__(self, coco, cocoRes):
self.evalImgs = []
self.eval = {}
self.imgToEval = {}
self.coco = coco
self.cocoRes = cocoRes
self.params = {'image_id': coco.getImgIds()}
self.Spice = Spice()
def evaluate(self):
imgIds = self.params['image_id']
# imgIds = self.coco.getImgIds()
gts = {}
res = {}
for imgId in imgIds:
gts[imgId] = self.coco.imgToAnns[imgId]
res[imgId] = self.cocoRes.imgToAnns[imgId]
# =================================================
# Set up scorers
# =================================================
print('tokenization...')
tokenizer = PTBTokenizer()
gts = tokenizer.tokenize(gts)
res = tokenizer.tokenize(res)
# =================================================
# Set up scorers
# =================================================
print('setting up scorers...')
scorers = [
(self.Spice, "SPICE")
]
# =================================================
# Compute scores
# =================================================
for scorer, method in scorers:
print('computing %s score...'%(scorer.method()))
score, scores = scorer.compute_score(gts, res)
if type(method) == list:
for sc, scs, m in zip(score, scores, method):
self.setEval(sc, m)
self.setImgToEvalImgs(scs, list(gts.keys()), m)
print("%s: %0.3f"%(m, sc))
else:
self.setEval(score, method)
self.setImgToEvalImgs(scores, list(gts.keys()), method)
print("%s: %0.3f"%(method, score))
self.setEvalImgs()
def setEval(self, score, method):
self.eval[method] = score
def setImgToEvalImgs(self, scores, imgIds, method):
for imgId, score in zip(sorted(imgIds), scores):
if not imgId in self.imgToEval:
self.imgToEval[imgId] = {}
self.imgToEval[imgId]["image_id"] = imgId
self.imgToEval[imgId][method] = score
def setEvalImgs(self):
self.evalImgs = [self.imgToEval[imgId] for imgId in sorted(self.imgToEval.keys())]
|