|
|
|
import os.path as osp |
|
from argparse import ArgumentParser |
|
|
|
import mmcv |
|
import numpy as np |
|
|
|
|
|
def print_coco_results(results): |
|
|
|
def _print(result, ap=1, iouThr=None, areaRng='all', maxDets=100): |
|
titleStr = 'Average Precision' if ap == 1 else 'Average Recall' |
|
typeStr = '(AP)' if ap == 1 else '(AR)' |
|
iouStr = '0.50:0.95' \ |
|
if iouThr is None else f'{iouThr:0.2f}' |
|
iStr = f' {titleStr:<18} {typeStr} @[ IoU={iouStr:<9} | ' |
|
iStr += f'area={areaRng:>6s} | maxDets={maxDets:>3d} ] = {result:0.3f}' |
|
print(iStr) |
|
|
|
stats = np.zeros((12, )) |
|
stats[0] = _print(results[0], 1) |
|
stats[1] = _print(results[1], 1, iouThr=.5) |
|
stats[2] = _print(results[2], 1, iouThr=.75) |
|
stats[3] = _print(results[3], 1, areaRng='small') |
|
stats[4] = _print(results[4], 1, areaRng='medium') |
|
stats[5] = _print(results[5], 1, areaRng='large') |
|
stats[6] = _print(results[6], 0, maxDets=1) |
|
stats[7] = _print(results[7], 0, maxDets=10) |
|
stats[8] = _print(results[8], 0) |
|
stats[9] = _print(results[9], 0, areaRng='small') |
|
stats[10] = _print(results[10], 0, areaRng='medium') |
|
stats[11] = _print(results[11], 0, areaRng='large') |
|
|
|
|
|
def get_coco_style_results(filename, |
|
task='bbox', |
|
metric=None, |
|
prints='mPC', |
|
aggregate='benchmark'): |
|
|
|
assert aggregate in ['benchmark', 'all'] |
|
|
|
if prints == 'all': |
|
prints = ['P', 'mPC', 'rPC'] |
|
elif isinstance(prints, str): |
|
prints = [prints] |
|
for p in prints: |
|
assert p in ['P', 'mPC', 'rPC'] |
|
|
|
if metric is None: |
|
metrics = [ |
|
'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100', |
|
'ARs', 'ARm', 'ARl' |
|
] |
|
elif isinstance(metric, list): |
|
metrics = metric |
|
else: |
|
metrics = [metric] |
|
|
|
for metric_name in metrics: |
|
assert metric_name in [ |
|
'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100', |
|
'ARs', 'ARm', 'ARl' |
|
] |
|
|
|
eval_output = mmcv.load(filename) |
|
|
|
num_distortions = len(list(eval_output.keys())) |
|
results = np.zeros((num_distortions, 6, len(metrics)), dtype='float32') |
|
|
|
for corr_i, distortion in enumerate(eval_output): |
|
for severity in eval_output[distortion]: |
|
for metric_j, metric_name in enumerate(metrics): |
|
mAP = eval_output[distortion][severity][task][metric_name] |
|
results[corr_i, severity, metric_j] = mAP |
|
|
|
P = results[0, 0, :] |
|
if aggregate == 'benchmark': |
|
mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) |
|
else: |
|
mPC = np.mean(results[:, 1:, :], axis=(0, 1)) |
|
rPC = mPC / P |
|
|
|
print(f'\nmodel: {osp.basename(filename)}') |
|
if metric is None: |
|
if 'P' in prints: |
|
print(f'Performance on Clean Data [P] ({task})') |
|
print_coco_results(P) |
|
if 'mPC' in prints: |
|
print(f'Mean Performance under Corruption [mPC] ({task})') |
|
print_coco_results(mPC) |
|
if 'rPC' in prints: |
|
print(f'Relative Performance under Corruption [rPC] ({task})') |
|
print_coco_results(rPC) |
|
else: |
|
if 'P' in prints: |
|
print(f'Performance on Clean Data [P] ({task})') |
|
for metric_i, metric_name in enumerate(metrics): |
|
print(f'{metric_name:5} = {P[metric_i]:0.3f}') |
|
if 'mPC' in prints: |
|
print(f'Mean Performance under Corruption [mPC] ({task})') |
|
for metric_i, metric_name in enumerate(metrics): |
|
print(f'{metric_name:5} = {mPC[metric_i]:0.3f}') |
|
if 'rPC' in prints: |
|
print(f'Relative Performance under Corruption [rPC] ({task})') |
|
for metric_i, metric_name in enumerate(metrics): |
|
print(f'{metric_name:5} => {rPC[metric_i] * 100:0.1f} %') |
|
|
|
return results |
|
|
|
|
|
def get_voc_style_results(filename, prints='mPC', aggregate='benchmark'): |
|
|
|
assert aggregate in ['benchmark', 'all'] |
|
|
|
if prints == 'all': |
|
prints = ['P', 'mPC', 'rPC'] |
|
elif isinstance(prints, str): |
|
prints = [prints] |
|
for p in prints: |
|
assert p in ['P', 'mPC', 'rPC'] |
|
|
|
eval_output = mmcv.load(filename) |
|
|
|
num_distortions = len(list(eval_output.keys())) |
|
results = np.zeros((num_distortions, 6, 20), dtype='float32') |
|
|
|
for i, distortion in enumerate(eval_output): |
|
for severity in eval_output[distortion]: |
|
mAP = [ |
|
eval_output[distortion][severity][j]['ap'] |
|
for j in range(len(eval_output[distortion][severity])) |
|
] |
|
results[i, severity, :] = mAP |
|
|
|
P = results[0, 0, :] |
|
if aggregate == 'benchmark': |
|
mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) |
|
else: |
|
mPC = np.mean(results[:, 1:, :], axis=(0, 1)) |
|
rPC = mPC / P |
|
|
|
print(f'\nmodel: {osp.basename(filename)}') |
|
if 'P' in prints: |
|
print(f'Performance on Clean Data [P] in AP50 = {np.mean(P):0.3f}') |
|
if 'mPC' in prints: |
|
print('Mean Performance under Corruption [mPC] in AP50 = ' |
|
f'{np.mean(mPC):0.3f}') |
|
if 'rPC' in prints: |
|
print('Relative Performance under Corruption [rPC] in % = ' |
|
f'{np.mean(rPC) * 100:0.1f}') |
|
|
|
return np.mean(results, axis=2, keepdims=True) |
|
|
|
|
|
def get_results(filename, |
|
dataset='coco', |
|
task='bbox', |
|
metric=None, |
|
prints='mPC', |
|
aggregate='benchmark'): |
|
assert dataset in ['coco', 'voc', 'cityscapes'] |
|
|
|
if dataset in ['coco', 'cityscapes']: |
|
results = get_coco_style_results( |
|
filename, |
|
task=task, |
|
metric=metric, |
|
prints=prints, |
|
aggregate=aggregate) |
|
elif dataset == 'voc': |
|
if task != 'bbox': |
|
print('Only bbox analysis is supported for Pascal VOC') |
|
print('Will report bbox results\n') |
|
if metric not in [None, ['AP'], ['AP50']]: |
|
print('Only the AP50 metric is supported for Pascal VOC') |
|
print('Will report AP50 metric\n') |
|
results = get_voc_style_results( |
|
filename, prints=prints, aggregate=aggregate) |
|
|
|
return results |
|
|
|
|
|
def get_distortions_from_file(filename): |
|
|
|
eval_output = mmcv.load(filename) |
|
|
|
return get_distortions_from_results(eval_output) |
|
|
|
|
|
def get_distortions_from_results(eval_output): |
|
distortions = [] |
|
for i, distortion in enumerate(eval_output): |
|
distortions.append(distortion.replace('_', ' ')) |
|
return distortions |
|
|
|
|
|
def main(): |
|
parser = ArgumentParser(description='Corruption Result Analysis') |
|
parser.add_argument('filename', help='result file path') |
|
parser.add_argument( |
|
'--dataset', |
|
type=str, |
|
choices=['coco', 'voc', 'cityscapes'], |
|
default='coco', |
|
help='dataset type') |
|
parser.add_argument( |
|
'--task', |
|
type=str, |
|
nargs='+', |
|
choices=['bbox', 'segm'], |
|
default=['bbox'], |
|
help='task to report') |
|
parser.add_argument( |
|
'--metric', |
|
nargs='+', |
|
choices=[ |
|
None, 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', |
|
'AR100', 'ARs', 'ARm', 'ARl' |
|
], |
|
default=None, |
|
help='metric to report') |
|
parser.add_argument( |
|
'--prints', |
|
type=str, |
|
nargs='+', |
|
choices=['P', 'mPC', 'rPC'], |
|
default='mPC', |
|
help='corruption benchmark metric to print') |
|
parser.add_argument( |
|
'--aggregate', |
|
type=str, |
|
choices=['all', 'benchmark'], |
|
default='benchmark', |
|
help='aggregate all results or only those \ |
|
for benchmark corruptions') |
|
|
|
args = parser.parse_args() |
|
|
|
for task in args.task: |
|
get_results( |
|
args.filename, |
|
dataset=args.dataset, |
|
task=task, |
|
metric=args.metric, |
|
prints=args.prints, |
|
aggregate=args.aggregate) |
|
|
|
|
|
if __name__ == '__main__': |
|
main() |
|
|
