Spaces:
Runtime error
Runtime error
| import torch | |
| import numpy as np | |
| import argparse | |
| from tqdm.autonotebook import tqdm | |
| import os | |
| from utils import smp_metrics | |
| from utils.utils import ConfusionMatrix, postprocess, scale_coords, process_batch, ap_per_class, fitness, \ | |
| save_checkpoint, DataLoaderX, BBoxTransform, ClipBoxes, boolean_string, Params | |
| from backbone import HybridNetsBackbone | |
| from hybridnets.dataset import BddDataset | |
| from torchvision import transforms | |
| def val(model, optimizer, val_generator, params, opt, writer, epoch, step, best_fitness, best_loss, best_epoch): | |
| model.eval() | |
| loss_regression_ls = [] | |
| loss_classification_ls = [] | |
| loss_segmentation_ls = [] | |
| jdict, stats, ap, ap_class = [], [], [], [] | |
| iou_thresholds = torch.linspace(0.5, 0.95, 10).cuda() # iou vector for [email protected]:0.95 | |
| num_thresholds = iou_thresholds.numel() | |
| names = {i: v for i, v in enumerate(params.obj_list)} | |
| nc = len(names) | |
| seen = 0 | |
| confusion_matrix = ConfusionMatrix(nc=nc) | |
| s = ('%15s' + '%11s' * 14) % ( | |
| 'Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95', 'mIoU', 'mF1', 'fIoU', 'sIoU', 'rIoU', 'rF1', 'lIoU', 'lF1') | |
| dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0, 0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 | |
| iou_ls = [[] for _ in range(3)] | |
| f1_ls = [[] for _ in range(3)] | |
| regressBoxes = BBoxTransform() | |
| clipBoxes = ClipBoxes() | |
| val_loader = tqdm(val_generator) | |
| for iter, data in enumerate(val_loader): | |
| imgs = data['img'] | |
| annot = data['annot'] | |
| seg_annot = data['segmentation'] | |
| filenames = data['filenames'] | |
| shapes = data['shapes'] | |
| if opt.num_gpus == 1: | |
| imgs = imgs.cuda() | |
| annot = annot.cuda() | |
| seg_annot = seg_annot.cuda() | |
| cls_loss, reg_loss, seg_loss, regression, classification, anchors, segmentation = model(imgs, annot, | |
| seg_annot, | |
| obj_list=params.obj_list) | |
| cls_loss = cls_loss.mean() | |
| reg_loss = reg_loss.mean() | |
| seg_loss = seg_loss.mean() | |
| if opt.cal_map: | |
| out = postprocess(imgs.detach(), | |
| torch.stack([anchors[0]] * imgs.shape[0], 0).detach(), regression.detach(), | |
| classification.detach(), | |
| regressBoxes, clipBoxes, | |
| 0.001, 0.6) # 0.5, 0.3 | |
| for i in range(annot.size(0)): | |
| seen += 1 | |
| labels = annot[i] | |
| labels = labels[labels[:, 4] != -1] | |
| ou = out[i] | |
| nl = len(labels) | |
| pred = np.column_stack([ou['rois'], ou['scores']]) | |
| pred = np.column_stack([pred, ou['class_ids']]) | |
| pred = torch.from_numpy(pred).cuda() | |
| target_class = labels[:, 4].tolist() if nl else [] # target class | |
| if len(pred) == 0: | |
| if nl: | |
| stats.append((torch.zeros(0, num_thresholds, dtype=torch.bool), | |
| torch.Tensor(), torch.Tensor(), target_class)) | |
| # print("here") | |
| continue | |
| if nl: | |
| pred[:, :4] = scale_coords(imgs[i][1:], pred[:, :4], shapes[i][0], shapes[i][1]) | |
| labels = scale_coords(imgs[i][1:], labels, shapes[i][0], shapes[i][1]) | |
| correct = process_batch(pred, labels, iou_thresholds) | |
| if opt.plots: | |
| confusion_matrix.process_batch(pred, labels) | |
| else: | |
| correct = torch.zeros(pred.shape[0], num_thresholds, dtype=torch.bool) | |
| stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), target_class)) | |
| # print(stats) | |
| # Visualization | |
| # seg_0 = segmentation[i] | |
| # # print('bbb', seg_0.shape) | |
| # seg_0 = torch.argmax(seg_0, dim = 0) | |
| # # print('before', seg_0.shape) | |
| # seg_0 = seg_0.cpu().numpy() | |
| # #.transpose(1, 2, 0) | |
| # # print(seg_0.shape) | |
| # anh = np.zeros((384,640,3)) | |
| # anh[seg_0 == 0] = (255,0,0) | |
| # anh[seg_0 == 1] = (0,255,0) | |
| # anh[seg_0 == 2] = (0,0,255) | |
| # anh = np.uint8(anh) | |
| # cv2.imwrite('segmentation-{}.jpg'.format(filenames[i]),anh) | |
| # Convert segmentation tensor --> 3 binary 0 1 | |
| # batch_size, num_classes, height, width | |
| _, segmentation = torch.max(segmentation, 1) | |
| # _, seg_annot = torch.max(seg_annot, 1) | |
| seg = torch.zeros((seg_annot.size(0), 3, 384, 640), dtype=torch.int32) | |
| seg[:, 0, ...][segmentation == 0] = 1 | |
| seg[:, 1, ...][segmentation == 1] = 1 | |
| seg[:, 2, ...][segmentation == 2] = 1 | |
| tp_seg, fp_seg, fn_seg, tn_seg = smp_metrics.get_stats(seg.cuda(), seg_annot.long().cuda(), | |
| mode='multilabel', threshold=None) | |
| iou = smp_metrics.iou_score(tp_seg, fp_seg, fn_seg, tn_seg, reduction='none') | |
| # print(iou) | |
| f1 = smp_metrics.balanced_accuracy(tp_seg, fp_seg, fn_seg, tn_seg, reduction='none') | |
| for i in range(len(params.seg_list) + 1): | |
| iou_ls[i].append(iou.T[i].detach().cpu().numpy()) | |
| f1_ls[i].append(f1.T[i].detach().cpu().numpy()) | |
| loss = cls_loss + reg_loss + seg_loss | |
| if loss == 0 or not torch.isfinite(loss): | |
| continue | |
| loss_classification_ls.append(cls_loss.item()) | |
| loss_regression_ls.append(reg_loss.item()) | |
| loss_segmentation_ls.append(seg_loss.item()) | |
| cls_loss = np.mean(loss_classification_ls) | |
| reg_loss = np.mean(loss_regression_ls) | |
| seg_loss = np.mean(loss_segmentation_ls) | |
| loss = cls_loss + reg_loss + seg_loss | |
| print( | |
| 'Val. Epoch: {}/{}. Classification loss: {:1.5f}. Regression loss: {:1.5f}. Segmentation loss: {:1.5f}. Total loss: {:1.5f}'.format( | |
| epoch, opt.num_epochs, cls_loss, reg_loss, seg_loss, loss)) | |
| writer.add_scalars('Loss', {'val': loss}, step) | |
| writer.add_scalars('Regression_loss', {'val': reg_loss}, step) | |
| writer.add_scalars('Classfication_loss', {'val': cls_loss}, step) | |
| writer.add_scalars('Segmentation_loss', {'val': seg_loss}, step) | |
| if opt.cal_map: | |
| # print(len(iou_ls[0])) | |
| iou_score = np.mean(iou_ls) | |
| # print(iou_score) | |
| f1_score = np.mean(f1_ls) | |
| iou_first_decoder = iou_ls[0] + iou_ls[1] | |
| iou_first_decoder = np.mean(iou_first_decoder) | |
| iou_second_decoder = iou_ls[0] + iou_ls[2] | |
| iou_second_decoder = np.mean(iou_second_decoder) | |
| for i in range(len(params.seg_list) + 1): | |
| iou_ls[i] = np.mean(iou_ls[i]) | |
| f1_ls[i] = np.mean(f1_ls[i]) | |
| # Compute statistics | |
| stats = [np.concatenate(x, 0) for x in zip(*stats)] | |
| # print(stats[3]) | |
| # Count detected boxes per class | |
| # boxes_per_class = np.bincount(stats[2].astype(np.int64), minlength=1) | |
| ap50 = None | |
| save_dir = 'plots' | |
| os.makedirs(save_dir, exist_ok=True) | |
| # Compute metrics | |
| if len(stats) and stats[0].any(): | |
| p, r, f1, ap, ap_class = ap_per_class(*stats, plot=opt.plots, save_dir=save_dir, names=names) | |
| ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 | |
| mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() | |
| nt = np.bincount(stats[3].astype(np.int64), minlength=1) # number of targets per class | |
| else: | |
| nt = torch.zeros(1) | |
| # Print results | |
| print(s) | |
| pf = '%15s' + '%11i' * 2 + '%11.3g' * 12 # print format | |
| print(pf % ('all', seen, nt.sum(), mp, mr, map50, map, iou_score, f1_score, iou_first_decoder, iou_second_decoder, | |
| iou_ls[1], f1_ls[1], iou_ls[2], f1_ls[2])) | |
| # Print results per class | |
| training = True | |
| if (opt.verbose or (nc < 50 and not training)) and nc > 1 and len(stats): | |
| for i, c in enumerate(ap_class): | |
| print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) | |
| # Plots | |
| if opt.plots: | |
| confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) | |
| confusion_matrix.tp_fp() | |
| results = (mp, mr, map50, map, iou_score, f1_score, loss) | |
| fi = fitness( | |
| np.array(results).reshape(1, -1)) # weighted combination of [P, R, [email protected], [email protected], iou, f1, loss ] | |
| # if calculating map, save by best fitness | |
| if fi > best_fitness: | |
| best_fitness = fi | |
| ckpt = {'epoch': epoch, | |
| 'step': step, | |
| 'best_fitness': best_fitness, | |
| 'model': model, | |
| 'optimizer': optimizer.state_dict()} | |
| print("Saving checkpoint with best fitness", fi[0]) | |
| save_checkpoint(ckpt, opt.saved_path, f'hybridnets-d{opt.compound_coef}_{epoch}_{step}_best.pth') | |
| else: | |
| # if not calculating map, save by best loss | |
| if loss + opt.es_min_delta < best_loss: | |
| best_loss = loss | |
| best_epoch = epoch | |
| save_checkpoint(model, opt.saved_path, f'hybridnets-d{opt.compound_coef}_{epoch}_{step}_best.pth') | |
| # Early stopping | |
| if epoch - best_epoch > opt.es_patience > 0: | |
| print('[Info] Stop training at epoch {}. The lowest loss achieved is {}'.format(epoch, best_loss)) | |
| writer.close() | |
| exit(0) | |
| model.train() | |
| return best_fitness, best_loss, best_epoch | |
| def val_from_cmd(model, val_generator, params, opt): | |
| model.eval() | |
| jdict, stats, ap, ap_class = [], [], [], [] | |
| iou_thresholds = torch.linspace(0.5, 0.95, 10).cuda() # iou vector for [email protected]:0.95 | |
| num_thresholds = iou_thresholds.numel() | |
| names = {i: v for i, v in enumerate(params.obj_list)} | |
| nc = len(names) | |
| seen = 0 | |
| confusion_matrix = ConfusionMatrix(nc=nc) | |
| s = ('%15s' + '%11s' * 14) % ( | |
| 'Class', 'Images', 'Labels', 'P', 'R', '[email protected]', '[email protected]:.95', 'mIoU', 'mF1', 'fIoU', 'sIoU', 'rIoU', 'rF1', 'lIoU', 'lF1') | |
| dt, p, r, f1, mp, mr, map50, map = [0.0, 0.0, 0.0], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 | |
| iou_ls = [[] for _ in range(3)] | |
| f1_ls = [[] for _ in range(3)] | |
| regressBoxes = BBoxTransform() | |
| clipBoxes = ClipBoxes() | |
| val_loader = tqdm(val_generator) | |
| for iter, data in enumerate(val_loader): | |
| imgs = data['img'] | |
| annot = data['annot'] | |
| seg_annot = data['segmentation'] | |
| filenames = data['filenames'] | |
| shapes = data['shapes'] | |
| if opt.num_gpus == 1: | |
| imgs = imgs.cuda() | |
| annot = annot.cuda() | |
| seg_annot = seg_annot.cuda() | |
| features, regressions, classifications, anchors, segmentation = model(imgs) | |
| out = postprocess(imgs.detach(), | |
| torch.stack([anchors[0]] * imgs.shape[0], 0).detach(), regressions.detach(), | |
| classifications.detach(), | |
| regressBoxes, clipBoxes, | |
| 0.001, 0.6) # 0.5, 0.3 | |
| # imgs = imgs.permute(0, 2, 3, 1).cpu().numpy() | |
| # imgs = ((imgs * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]) * 255).astype(np.uint8) | |
| # imgs = [cv2.cvtColor(img, cv2.COLOR_RGB2BGR) for img in imgs] | |
| # display(out, imgs, ['car'], imshow=False, imwrite=True) | |
| # for index, filename in enumerate(filenames): | |
| # ori_img = cv2.imread('datasets/bdd100k/val/'+filename) | |
| # if len(out[index]['rois']): | |
| # for roi in out[index]['rois']: | |
| # x1,y1,x2,y2 = [int(x) for x in roi] | |
| # cv2.rectangle(ori_img, (x1,y1), (x2,y2), (255,0,0), 1) | |
| # cv2.imwrite(filename, ori_img) | |
| for i in range(annot.size(0)): | |
| seen += 1 | |
| labels = annot[i] | |
| labels = labels[labels[:, 4] != -1] | |
| ou = out[i] | |
| nl = len(labels) | |
| pred = np.column_stack([ou['rois'], ou['scores']]) | |
| pred = np.column_stack([pred, ou['class_ids']]) | |
| pred = torch.from_numpy(pred).cuda() | |
| target_class = labels[:, 4].tolist() if nl else [] # target class | |
| if len(pred) == 0: | |
| if nl: | |
| stats.append((torch.zeros(0, num_thresholds, dtype=torch.bool), | |
| torch.Tensor(), torch.Tensor(), target_class)) | |
| # print("here") | |
| continue | |
| if nl: | |
| pred[:, :4] = scale_coords(imgs[i][1:], pred[:, :4], shapes[i][0], shapes[i][1]) | |
| labels = scale_coords(imgs[i][1:], labels, shapes[i][0], shapes[i][1]) | |
| # ori_img = cv2.imread('datasets/bdd100k_effdet/val/' + filenames[i], | |
| # cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION | cv2.IMREAD_UNCHANGED) | |
| # for label in labels: | |
| # x1, y1, x2, y2 = [int(x) for x in label[:4]] | |
| # ori_img = cv2.rectangle(ori_img, (x1, y1), (x2, y2), (255, 0, 0), 1) | |
| # for pre in pred: | |
| # x1, y1, x2, y2 = [int(x) for x in pre[:4]] | |
| # # ori_img = cv2.putText(ori_img, str(pre[4].cpu().numpy()), (x1 - 10, y1 - 10), | |
| # # cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1, cv2.LINE_AA) | |
| # ori_img = cv2.rectangle(ori_img, (x1, y1), (x2, y2), (0, 255, 0), 1) | |
| # cv2.imwrite('pre+label-{}.jpg'.format(filenames[i]), ori_img) | |
| correct = process_batch(pred, labels, iou_thresholds) | |
| if opt.plots: | |
| confusion_matrix.process_batch(pred, labels) | |
| else: | |
| correct = torch.zeros(pred.shape[0], num_thresholds, dtype=torch.bool) | |
| stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), target_class)) | |
| # print(stats) | |
| # Visualization | |
| # seg_0 = segmentation[i] | |
| # # print('bbb', seg_0.shape) | |
| # seg_0 = torch.argmax(seg_0, dim = 0) | |
| # # print('before', seg_0.shape) | |
| # seg_0 = seg_0.cpu().numpy() | |
| # #.transpose(1, 2, 0) | |
| # # print(seg_0.shape) | |
| # anh = np.zeros((384,640,3)) | |
| # anh[seg_0 == 0] = (255,0,0) | |
| # anh[seg_0 == 1] = (0,255,0) | |
| # anh[seg_0 == 2] = (0,0,255) | |
| # anh = np.uint8(anh) | |
| # cv2.imwrite('segmentation-{}.jpg'.format(filenames[i]),anh) | |
| # Convert segmentation tensor --> 3 binary 0 1 | |
| # batch_size, num_classes, height, width | |
| _, segmentation = torch.max(segmentation, 1) | |
| # _, seg_annot = torch.max(seg_annot, 1) | |
| seg = torch.zeros((seg_annot.size(0), 3, 384, 640), dtype=torch.int32) | |
| seg[:, 0, ...][segmentation == 0] = 1 | |
| seg[:, 1, ...][segmentation == 1] = 1 | |
| seg[:, 2, ...][segmentation == 2] = 1 | |
| tp_seg, fp_seg, fn_seg, tn_seg = smp_metrics.get_stats(seg.cuda(), seg_annot.long().cuda(), mode='multilabel', | |
| threshold=None) | |
| iou = smp_metrics.iou_score(tp_seg, fp_seg, fn_seg, tn_seg, reduction='none') | |
| # print(iou) | |
| f1 = smp_metrics.balanced_accuracy(tp_seg, fp_seg, fn_seg, tn_seg, reduction='none') | |
| for i in range(len(params.seg_list) + 1): | |
| iou_ls[i].append(iou.T[i].detach().cpu().numpy()) | |
| f1_ls[i].append(f1.T[i].detach().cpu().numpy()) | |
| # Visualize | |
| # for i in range(segmentation.size(0)): | |
| # if iou_ls[1][iter][i] < 0.4: | |
| # import cv2 | |
| # | |
| # ori = cv2.imread('datasets/bdd100k/val/{}'.format(filenames[i])) | |
| # cv2.imwrite('ori-segmentation-{}-{}.jpg'.format(iter,filenames[i]),ori) | |
| # | |
| # gt = seg_annot[i].detach() | |
| # gt = torch.argmax(gt, dim = 0).cpu().numpy() | |
| # | |
| # anh = np.zeros((384,640,3)) | |
| # anh[gt == 0] = (255,0,0) | |
| # anh[gt == 1] = (0,255,0) | |
| # anh[gt == 2] = (0,0,255) | |
| # cv2.imwrite('gt-segmentation-{}-{}.jpg'.format(iter,filenames[i]),anh) | |
| # | |
| # seg_0 = seg[i] | |
| # seg_0 = torch.argmax(seg_0, dim = 0) | |
| # seg_0 = seg_0.cpu().numpy() | |
| # anh = np.zeros((384,640,3)) | |
| # anh[seg_0 == 0] = (255,0,0) | |
| # anh[seg_0 == 1] = (0,255,0) | |
| # anh[seg_0 == 2] = (0,0,255) | |
| # anh = np.uint8(anh) | |
| # cv2.imwrite('segmentation-{}-{}.jpg'.format(iter,filenames[i]),anh) | |
| # print(len(iou_ls[0])) | |
| # print(iou_ls) | |
| iou_score = np.mean(iou_ls) | |
| # print(iou_score) | |
| f1_score = np.mean(f1_ls) | |
| iou_first_decoder = iou_ls[0] + iou_ls[1] | |
| iou_first_decoder = np.mean(iou_first_decoder) | |
| iou_second_decoder = iou_ls[0] + iou_ls[2] | |
| iou_second_decoder = np.mean(iou_second_decoder) | |
| for i in range(len(params.seg_list) + 1): | |
| iou_ls[i] = np.mean(iou_ls[i]) | |
| f1_ls[i] = np.mean(f1_ls[i]) | |
| # Compute statistics | |
| stats = [np.concatenate(x, 0) for x in zip(*stats)] | |
| # Count detected boxes per class | |
| # boxes_per_class = np.bincount(stats[2].astype(np.int64), minlength=1) | |
| ap50 = None | |
| save_dir = 'plots' | |
| os.makedirs(save_dir, exist_ok=True) | |
| # Compute metrics | |
| if len(stats) and stats[0].any(): | |
| p, r, f1, ap, ap_class = ap_per_class(*stats, plot=opt.plots, save_dir=save_dir, names=names) | |
| ap50, ap = ap[:, 0], ap.mean(1) # [email protected], [email protected]:0.95 | |
| mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() | |
| nt = np.bincount(stats[3].astype(np.int64), minlength=1) # number of targets per class | |
| else: | |
| nt = torch.zeros(1) | |
| # Print results | |
| print(s) | |
| pf = '%15s' + '%11i' * 2 + '%11.3g' * 12 # print format | |
| print(pf % ('all', seen, nt.sum(), mp, mr, map50, map, iou_score, f1_score, iou_first_decoder, iou_second_decoder, | |
| iou_ls[1], f1_ls[1], iou_ls[2], f1_ls[2])) | |
| # Print results per class | |
| training = False | |
| if (opt.verbose or (nc < 50 and not training)) and nc > 1 and len(stats): | |
| for i, c in enumerate(ap_class): | |
| print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) | |
| # Plots | |
| if opt.plots: | |
| confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) | |
| confusion_matrix.tp_fp() | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument('-p', '--project', type=str, default='coco', help='Project file that contains parameters') | |
| ap.add_argument('-c', '--compound_coef', type=int, default=0, help='Coefficients of efficientnet backbone') | |
| ap.add_argument('-w', '--weights', type=str, default=None, help='/path/to/weights') | |
| ap.add_argument('-n', '--num_workers', type=int, default=12, help='Num_workers of dataloader') | |
| ap.add_argument('--batch_size', type=int, default=12, help='The number of images per batch among all devices') | |
| ap.add_argument('-v', '--verbose', type=boolean_string, default=True, | |
| help='Whether to print results per class when valing') | |
| ap.add_argument('--plots', type=boolean_string, default=True, | |
| help='Whether to plot confusion matrix when valing') | |
| ap.add_argument('--num_gpus', type=int, default=1, | |
| help='Number of GPUs to be used (0 to use CPU)') | |
| args = ap.parse_args() | |
| compound_coef = args.compound_coef | |
| project_name = args.project | |
| weights_path = f'weights/hybridnets-d{compound_coef}.pth' if args.weights is None else args.weights | |
| params = Params(f'projects/{project_name}.yml') | |
| obj_list = params.obj_list | |
| valid_dataset = BddDataset( | |
| params=params, | |
| is_train=False, | |
| inputsize=params.model['image_size'], | |
| transform=transforms.Compose([ | |
| transforms.ToTensor(), | |
| transforms.Normalize( | |
| mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] | |
| ) | |
| ]) | |
| ) | |
| val_generator = DataLoaderX( | |
| valid_dataset, | |
| batch_size=args.batch_size, | |
| shuffle=False, | |
| num_workers=args.num_workers, | |
| pin_memory=params.pin_memory, | |
| collate_fn=BddDataset.collate_fn | |
| ) | |
| model = HybridNetsBackbone(compound_coef=compound_coef, num_classes=len(params.obj_list), | |
| ratios=eval(params.anchors_ratios), scales=eval(params.anchors_scales), | |
| seg_classes=len(params.seg_list)) | |
| # print(model) | |
| try: | |
| model.load_state_dict(torch.load(weights_path)) | |
| except: | |
| model.load_state_dict(torch.load(weights_path)['model']) | |
| model.requires_grad_(False) | |
| if args.num_gpus > 0: | |
| model.cuda() | |
| val_from_cmd(model, val_generator, params, args) | |