import torch import torch.nn as nn import render_util import geo_transform import numpy as np def compute_tri_normal(geometry, tris): geometry = geometry.permute(0, 2, 1) tri_1 = tris[:, 0] tri_2 = tris[:, 1] tri_3 = tris[:, 2] vert_1 = torch.index_select(geometry, 2, tri_1) vert_2 = torch.index_select(geometry, 2, tri_2) vert_3 = torch.index_select(geometry, 2, tri_3) nnorm = torch.cross(vert_2 - vert_1, vert_3 - vert_1, 1) normal = nn.functional.normalize(nnorm).permute(0, 2, 1) return normal class Compute_normal_base(torch.autograd.Function): @staticmethod def forward(ctx, normal): (normal_b,) = render_util.normal_base_forward(normal) ctx.save_for_backward(normal) return normal_b @staticmethod def backward(ctx, grad_normal_b): (normal,) = ctx.saved_tensors (grad_normal,) = render_util.normal_base_backward(grad_normal_b, normal) return grad_normal class Normal_Base(torch.nn.Module): def __init__(self): super(Normal_Base, self).__init__() def forward(self, normal): return Compute_normal_base.apply(normal) def preprocess_render(geometry, euler, trans, cam, tris, vert_tris, ori_img): point_num = geometry.shape[1] rott_geo = geo_transform.euler_trans_geo(geometry, euler, trans) proj_geo = geo_transform.proj_geo(rott_geo, cam) rot_tri_normal = compute_tri_normal(rott_geo, tris) rot_vert_normal = torch.index_select(rot_tri_normal, 1, vert_tris) is_visible = -torch.bmm( rot_vert_normal.reshape(-1, 1, 3), nn.functional.normalize(rott_geo.reshape(-1, 3, 1)), ).reshape(-1, point_num) is_visible[is_visible < 0.01] = -1 pixel_valid = torch.zeros( (ori_img.shape[0], ori_img.shape[1] * ori_img.shape[2]), dtype=torch.float32, device=ori_img.device, ) return rott_geo, proj_geo, rot_tri_normal, is_visible, pixel_valid class Render_Face(torch.autograd.Function): @staticmethod def forward( ctx, proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid ): batch_size, h, w, _ = ori_img.shape ori_img = ori_img.view(batch_size, -1, 3) ori_size = torch.cat( ( torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device) * h, torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device) * w, ), dim=1, ).view(-1) tri_index, tri_coord, render, real = render_util.render_face_forward( proj_geo, ori_img, ori_size, texture, nbl, is_visible, tri_inds, pixel_valid ) ctx.save_for_backward( ori_img, ori_size, proj_geo, texture, nbl, tri_inds, tri_index, tri_coord ) return render, real @staticmethod def backward(ctx, grad_render, grad_real): ( ori_img, ori_size, proj_geo, texture, nbl, tri_inds, tri_index, tri_coord, ) = ctx.saved_tensors grad_proj_geo, grad_texture, grad_nbl = render_util.render_face_backward( grad_render, grad_real, ori_img, ori_size, proj_geo, texture, nbl, tri_inds, tri_index, tri_coord, ) return grad_proj_geo, grad_texture, grad_nbl, None, None, None, None class Render_RGB(nn.Module): def __init__(self): super(Render_RGB, self).__init__() def forward( self, proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid ): return Render_Face.apply( proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid ) def cal_land(proj_geo, is_visible, lands_info, land_num): (land_index,) = render_util.update_contour(lands_info, is_visible, land_num) proj_land = torch.index_select(proj_geo.reshape(-1, 3), 0, land_index)[ :, :2 ].reshape(-1, land_num, 2) return proj_land class Render_Land(nn.Module): def __init__(self): super(Render_Land, self).__init__() lands_info = np.loadtxt("../data/3DMM/lands_info.txt", dtype=np.int32) self.lands_info = torch.as_tensor(lands_info).cuda() tris = np.loadtxt("../data/3DMM/tris.txt", dtype=np.int64) self.tris = torch.as_tensor(tris).cuda() - 1 vert_tris = np.loadtxt("../data/3DMM/vert_tris.txt", dtype=np.int64) self.vert_tris = torch.as_tensor(vert_tris).cuda() self.normal_baser = Normal_Base().cuda() self.renderer = Render_RGB().cuda() def render_mesh(self, geometry, euler, trans, cam, ori_img, light): batch_size, h, w, _ = ori_img.shape ori_img = ori_img.view(batch_size, -1, 3) ori_size = torch.cat( ( torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device) * h, torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device) * w, ), dim=1, ).view(-1) rott_geo, proj_geo, rot_tri_normal, _, _ = preprocess_render( geometry, euler, trans, cam, self.tris, self.vert_tris, ori_img ) tri_nb = self.normal_baser(rot_tri_normal.contiguous()) nbl = torch.bmm( tri_nb, (light.reshape(-1, 9, 3))[:, :, 0].unsqueeze(-1).repeat(1, 1, 3) ) texture = torch.ones_like(geometry) * 200 (render,) = render_util.render_mesh( proj_geo, ori_img, ori_size, texture, nbl, self.tris ) return render.view(batch_size, h, w, 3).byte() def cal_loss_rgb(self, geometry, euler, trans, cam, ori_img, light, texture, lands): rott_geo, proj_geo, rot_tri_normal, is_visible, pixel_valid = preprocess_render( geometry, euler, trans, cam, self.tris, self.vert_tris, ori_img ) tri_nb = self.normal_baser(rot_tri_normal.contiguous()) nbl = torch.bmm(tri_nb, light.reshape(-1, 9, 3)) render, real = self.renderer( proj_geo, texture, nbl, ori_img, is_visible, self.tris, pixel_valid ) proj_land = cal_land(proj_geo, is_visible, self.lands_info, lands.shape[1]) col_minus = torch.norm((render - real).reshape(-1, 3), dim=1).reshape( ori_img.shape[0], -1 ) col_dis = torch.mean(col_minus * pixel_valid) / ( torch.mean(pixel_valid) + 0.00001 ) land_dists = torch.norm((proj_land - lands).reshape(-1, 2), dim=1).reshape( ori_img.shape[0], -1 ) lan_dis = torch.mean(land_dists) return col_dis, lan_dis