Upload 19 files

app.py

@@ -0,0 +1,49 @@
+import gradio as gr
+import os
+
+import pytorch_lightning as pl
+import torch as th
+import open3d as o3d
+import numpy as np
+import trimesh as tm
+
+from models.model import Model
+
+model = Model()
+ckpg = th.load("./checkpoints/epoch=99-step=6000.ckpt")
+model.load_state_dict(ckpg["state_dict"])
+
+
+def process_mesh(mesh_file_name):
+
+ mesh = tm.load_mesh(mesh_file_name)
+
+ v = th.tensor(mesh.vertices, dtype=th.float)
+ n = th.tensor(mesh.vertex_normals, dtype=th.float)
+
+ with th.no_grad():
+ v, f, n, _ = model(v.unsqueeze(0), n.unsqueeze(0))
+
+ mesh = tm.Trimesh(vertices=v.squeeze(0), 
+ faces=f.squeeze(0), 
+ vertex_normals=n.squeeze(0))
+ obj_path = "./sample.obj"
+ mesh.export(obj_path)
+
+ return obj_path
+
+
+demo = gr.Interface(
+ fn=process_mesh,
+ inputs=gr.Model3D(),
+ outputs=gr.Model3D(
+ clear_color=[0.0, 0.0, 0.0, 0.0], label="3D Model"),
+ examples=[
+ [os.path.join(os.path.dirname(__file__), "files\\bunny_n1_hi_50.obj")],
+ [os.path.join(os.path.dirname(__file__), "files\\child_n2_80.obj")],
+ [os.path.join(os.path.dirname(__file__), "files\\eight_n3_70.obj")],
+ ],
+)
+
+if __name__ == "__main__":
+ demo.launch()

checkpoints/epoch=99-step=6000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3a025dfa88edbf34bf7cf2b69c554cb01ab7d61b4d8cc699a2a6753e14dbdea
+size 4308343

models/SAP/dpsr.py

@@ -0,0 +1,65 @@
+import torch
+import torch.nn as nn
+from .utils import spec_gaussian_filter, fftfreqs, img, grid_interp, point_rasterize
+import numpy as np
+import torch.fft
+
+class DPSR(nn.Module):
+ def __init__(self, res, sig=10, scale=True, shift=True):
+ """
+ :param res: tuple of output field resolution. eg., (128,128)
+ :param sig: degree of gaussian smoothing
+ """
+ super(DPSR, self).__init__()
+ self.res = res
+ self.sig = sig
+ self.dim = len(res)
+ self.denom = np.prod(res)
+ G = spec_gaussian_filter(res=res, sig=sig).float()
+ # self.G.requires_grad = False # True, if we also make sig a learnable parameter
+ self.omega = fftfreqs(res, dtype=torch.float32)
+ self.scale = scale
+ self.shift = shift
+ self.register_buffer("G", G)
+ 
+ def forward(self, V, N):
+ """
+ :param V: (batch, nv, 2 or 3) tensor for point cloud coordinates
+ :param N: (batch, nv, 2 or 3) tensor for point normals
+ :return phi: (batch, res, res, ...) tensor of output indicator function field
+ """
+ assert(V.shape == N.shape) # [b, nv, ndims]
+ ras_p = point_rasterize(V, N, self.res) # [b, n_dim, dim0, dim1, dim2]
+ 
+ ras_s = torch.fft.rfftn(ras_p, dim=(2,3,4))
+ ras_s = ras_s.permute(*tuple([0]+list(range(2, self.dim+1))+[self.dim+1, 1]))
+ N_ = ras_s[..., None] * self.G # [b, dim0, dim1, dim2/2+1, n_dim, 1]
+
+ omega = fftfreqs(self.res, dtype=torch.float32).unsqueeze(-1) # [dim0, dim1, dim2/2+1, n_dim, 1]
+ omega *= 2 * np.pi # normalize frequencies
+ omega = omega.to(V.device)
+ 
+ DivN = torch.sum(-img(torch.view_as_real(N_[..., 0])) * omega, dim=-2)
+ 
+ Lap = -torch.sum(omega**2, -2) # [dim0, dim1, dim2/2+1, 1]
+ Phi = DivN / (Lap+1e-6) # [b, dim0, dim1, dim2/2+1, 2] 
+ Phi = Phi.permute(*tuple([list(range(1,self.dim+2)) + [0]])) # [dim0, dim1, dim2/2+1, 2, b] 
+ Phi[tuple([0] * self.dim)] = 0
+ Phi = Phi.permute(*tuple([[self.dim+1] + list(range(self.dim+1))])) # [b, dim0, dim1, dim2/2+1, 2]
+ 
+ phi = torch.fft.irfftn(torch.view_as_complex(Phi), s=self.res, dim=(1,2,3))
+ 
+ if self.shift or self.scale:
+ # ensure values at points are zero
+ fv = grid_interp(phi.unsqueeze(-1), V, batched=True).squeeze(-1) # [b, nv]
+ if self.shift: # offset points to have mean of 0
+ offset = torch.mean(fv, dim=-1) # [b,] 
+ phi -= offset.view(*tuple([-1] + [1] * self.dim))
+ 
+ phi = phi.permute(*tuple([list(range(1,self.dim+1)) + [0]]))
+ fv0 = phi[tuple([0] * self.dim)] # [b,]
+ phi = phi.permute(*tuple([[self.dim] + list(range(self.dim))]))
+ 
+ if self.scale:
+ phi = -phi / torch.abs(fv0.view(*tuple([-1]+[1] * self.dim))) *0.5
+ return phi

models/SAP/model.py

@@ -0,0 +1,129 @@
+import torch
+import numpy as np
+import time
+from .utils import point_rasterize, grid_interp, mc_from_psr, \
+calc_inters_points
+from .dpsr import DPSR
+import torch.nn as nn
+
+class PSR2Mesh(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, psr_grid):
+ """
+ In the forward pass we receive a Tensor containing the input and return
+ a Tensor containing the output. ctx is a context object that can be used
+ to stash information for backward computation. You can cache arbitrary
+ objects for use in the backward pass using the ctx.save_for_backward method.
+ """
+ verts, faces, normals = mc_from_psr(psr_grid, pytorchify=True)
+ verts = verts.unsqueeze(0)
+ faces = faces.unsqueeze(0)
+ normals = normals.unsqueeze(0)
+
+ res = torch.tensor(psr_grid.detach().shape[2])
+ ctx.save_for_backward(verts, normals, res)
+
+ return verts, faces, normals
+
+ @staticmethod
+ def backward(ctx, dL_dVertex, dL_dFace, dL_dNormals):
+ """
+ In the backward pass we receive a Tensor containing the gradient of the loss
+ with respect to the output, and we need to compute the gradient of the loss
+ with respect to the input.
+ """
+ vert_pts, normals, res = ctx.saved_tensors
+ res = (res.item(), res.item(), res.item())
+ # matrix multiplication between dL/dV and dV/dPSR
+ # dV/dPSR = - normals
+ grad_vert = torch.matmul(dL_dVertex.permute(1, 0, 2), -normals.permute(1, 2, 0))
+ grad_grid = point_rasterize(vert_pts, grad_vert.permute(1, 0, 2), res) # b x 1 x res x res x res
+ 
+ return grad_grid
+
+class PSR2SurfacePoints(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, psr_grid, poses, img_size, uv, psr_grad, mask_sample):
+ verts, faces, normals = mc_from_psr(psr_grid, pytorchify=True)
+ verts = verts * 2. - 1. # within the range of [-1, 1]
+
+ 
+ p_all, n_all, mask_all = [], [], []
+
+ for i in range(len(poses)):
+ pose = poses[i]
+ if mask_sample is not None:
+ p_inters, mask, _, _ = calc_inters_points(verts, faces, pose, img_size, mask_gt=mask_sample[i])
+ else:
+ p_inters, mask, _, _ = calc_inters_points(verts, faces, pose, img_size)
+
+ n_inters = grid_interp(psr_grad[None], (p_inters[None].detach() + 1) / 2).squeeze()
+ p_all.append(p_inters)
+ n_all.append(n_inters)
+ mask_all.append(mask)
+ p_inters_all = torch.cat(p_all, dim=0)
+ n_inters_all = torch.cat(n_all, dim=0)
+ mask_visible = torch.stack(mask_all, dim=0)
+
+
+ res = torch.tensor(psr_grid.detach().shape[2])
+ ctx.save_for_backward(p_inters_all, n_inters_all, res)
+
+ return p_inters_all, mask_visible
+
+ @staticmethod
+ def backward(ctx, dL_dp, dL_dmask):
+ pts, pts_n, res = ctx.saved_tensors
+ res = (res.item(), res.item(), res.item())
+
+ # grad from the p_inters via MLP renderer
+ grad_pts = torch.matmul(dL_dp[:, None], -pts_n[..., None])
+ grad_grid_pts = point_rasterize((pts[None]+1)/2, grad_pts.permute(1, 0, 2), res) # b x 1 x res x res x res
+ 
+ return grad_grid_pts, None, None, None, None, None
+
+ 
+# Resnet Blocks from https://github.com/autonomousvision/shape_as_points/blob/12757682f1075d83738b52f96747463b77343caf/src/network/utils.py
+class ResnetBlockFC(nn.Module):
+ ''' Fully connected ResNet Block class.
+ Args:
+ size_in (int): input dimension
+ size_out (int): output dimension
+ size_h (int): hidden dimension
+ '''
+
+ def __init__(self, size_in, size_out=None, size_h=None, siren=False):
+ super().__init__()
+ # Attributes
+ if size_out is None:
+ size_out = size_in
+
+ if size_h is None:
+ size_h = min(size_in, size_out)
+
+ self.size_in = size_in
+ self.size_h = size_h
+ self.size_out = size_out
+ # Submodules
+ self.fc_0 = nn.Linear(size_in, size_h)
+ self.fc_1 = nn.Linear(size_h, size_out)
+ self.actvn = nn.ReLU()
+
+ if size_in == size_out:
+ self.shortcut = None
+ else:
+ self.shortcut = nn.Linear(size_in, size_out, bias=False)
+ # Initialization
+ nn.init.zeros_(self.fc_1.weight)
+
+ def forward(self, x):
+ net = self.fc_0(self.actvn(x))
+ dx = self.fc_1(self.actvn(net))
+
+ if self.shortcut is not None:
+ x_s = self.shortcut(x)
+ else:
+ x_s = x
+
+ return x_s + dx
+ 

models/SAP/utils.py

@@ -0,0 +1,526 @@
+import torch
+import io, os, logging, urllib
+import yaml
+import trimesh
+import imageio
+import numbers
+import math
+import numpy as np
+from collections import OrderedDict
+from plyfile import PlyData
+from torch import nn
+from torch.nn import functional as F
+from torch.utils import model_zoo
+from skimage import measure, img_as_float32
+from igl import adjacency_matrix, connected_components
+
+##################################################
+# Below are functions for DPSR
+
+def fftfreqs(res, dtype=torch.float32, exact=True):
+ """
+ Helper function to return frequency tensors
+ :param res: n_dims int tuple of number of frequency modes
+ :return:
+ """
+
+ n_dims = len(res)
+ freqs = []
+ for dim in range(n_dims - 1):
+ r_ = res[dim]
+ freq = np.fft.fftfreq(r_, d=1/r_)
+ freqs.append(torch.tensor(freq, dtype=dtype))
+ r_ = res[-1]
+ if exact:
+ freqs.append(torch.tensor(np.fft.rfftfreq(r_, d=1/r_), dtype=dtype))
+ else:
+ freqs.append(torch.tensor(np.fft.rfftfreq(r_, d=1/r_)[:-1], dtype=dtype))
+ omega = torch.meshgrid(freqs)
+ omega = list(omega)
+ omega = torch.stack(omega, dim=-1)
+
+ return omega
+
+def img(x, deg=1): # imaginary of tensor (assume last dim: real/imag)
+ """
+ multiply tensor x by i ** deg
+ """
+ deg %= 4
+ if deg == 0:
+ res = x
+ elif deg == 1:
+ res = x[..., [1, 0]]
+ res[..., 0] = -res[..., 0]
+ elif deg == 2:
+ res = -x
+ elif deg == 3:
+ res = x[..., [1, 0]]
+ res[..., 1] = -res[..., 1]
+ return res
+
+def spec_gaussian_filter(res, sig):
+ omega = fftfreqs(res, dtype=torch.float64) # [dim0, dim1, dim2, d]
+ dis = torch.sqrt(torch.sum(omega ** 2, dim=-1))
+ filter_ = torch.exp(-0.5*((sig*2*dis/res[0])**2)).unsqueeze(-1).unsqueeze(-1)
+ filter_.requires_grad = False
+
+ return filter_
+
+def grid_interp(grid, pts, batched=True):
+ """
+ :param grid: tensor of shape (batch, *size, in_features)
+ :param pts: tensor of shape (batch, num_points, dim) within range (0, 1)
+ :return values at query points
+ """
+ if not batched:
+ grid = grid.unsqueeze(0)
+ pts = pts.unsqueeze(0)
+ dim = pts.shape[-1]
+ bs = grid.shape[0]
+ size = torch.tensor(grid.shape[1:-1]).to(grid.device).type(pts.dtype)
+ cubesize = 1.0 / size
+ 
+ ind0 = torch.floor(pts / cubesize).long() # (batch, num_points, dim)
+ ind1 = torch.fmod(torch.ceil(pts / cubesize), size).long() # periodic wrap-around
+ ind01 = torch.stack((ind0, ind1), dim=0) # (2, batch, num_points, dim)
+ tmp = torch.tensor([0,1],dtype=torch.long)
+ com_ = torch.stack(torch.meshgrid(tuple([tmp] * dim)), dim=-1).view(-1, dim)
+ dim_ = torch.arange(dim).repeat(com_.shape[0], 1) # (2**dim, dim)
+ ind_ = ind01[com_, ..., dim_] # (2**dim, dim, batch, num_points)
+ ind_n = ind_.permute(2, 3, 0, 1) # (batch, num_points, 2**dim, dim)
+ ind_b = torch.arange(bs).expand(ind_n.shape[1], ind_n.shape[2], bs).permute(2, 0, 1) # (batch, num_points, 2**dim)
+ # latent code on neighbor nodes
+ if dim == 2:
+ lat = grid.clone()[ind_b, ind_n[..., 0], ind_n[..., 1]] # (batch, num_points, 2**dim, in_features)
+ else:
+ lat = grid.clone()[ind_b, ind_n[..., 0], ind_n[..., 1], ind_n[..., 2]] # (batch, num_points, 2**dim, in_features)
+
+ # weights of neighboring nodes
+ xyz0 = ind0.type(cubesize.dtype) * cubesize # (batch, num_points, dim)
+ xyz1 = (ind0.type(cubesize.dtype) + 1) * cubesize # (batch, num_points, dim)
+ xyz01 = torch.stack((xyz0, xyz1), dim=0) # (2, batch, num_points, dim)
+ pos = xyz01[com_, ..., dim_].permute(2,3,0,1) # (batch, num_points, 2**dim, dim)
+ pos_ = xyz01[1-com_, ..., dim_].permute(2,3,0,1) # (batch, num_points, 2**dim, dim)
+ pos_ = pos_.type(pts.dtype)
+ dxyz_ = torch.abs(pts.unsqueeze(-2) - pos_) / cubesize # (batch, num_points, 2**dim, dim)
+ weights = torch.prod(dxyz_, dim=-1, keepdim=False) # (batch, num_points, 2**dim)
+ query_values = torch.sum(lat * weights.unsqueeze(-1), dim=-2) # (batch, num_points, in_features)
+ if not batched:
+ query_values = query_values.squeeze(0)
+ 
+ return query_values
+
+def scatter_to_grid(inds, vals, size):
+ """
+ Scatter update values into empty tensor of size size.
+ :param inds: (#values, dims)
+ :param vals: (#values)
+ :param size: tuple for size. len(size)=dims
+ """
+ dims = inds.shape[1]
+ assert(inds.shape[0] == vals.shape[0])
+ assert(len(size) == dims)
+ dev = vals.device
+ # result = torch.zeros(*size).view(-1).to(dev).type(vals.dtype) # flatten
+ # # flatten inds
+ result = torch.zeros(*size, device=dev).view(-1).type(vals.dtype) # flatten
+ # flatten inds
+ fac = [np.prod(size[i+1:]) for i in range(len(size)-1)] + [1]
+ fac = torch.tensor(fac, device=dev).type(inds.dtype)
+ inds_fold = torch.sum(inds*fac, dim=-1) # [#values,]
+ result.scatter_add_(0, inds_fold, vals)
+ result = result.view(*size)
+ return result
+
+def point_rasterize(pts, vals, size):
+ """
+ :param pts: point coords, tensor of shape (batch, num_points, dim) within range (0, 1)
+ :param vals: point values, tensor of shape (batch, num_points, features)
+ :param size: len(size)=dim tuple for grid size
+ :return rasterized values (batch, features, res0, res1, res2)
+ """
+ dim = pts.shape[-1]
+ assert(pts.shape[:2] == vals.shape[:2])
+ assert(pts.shape[2] == dim)
+ size_list = list(size)
+ size = torch.tensor(size).to(pts.device).float()
+ cubesize = 1.0 / size
+ bs = pts.shape[0]
+ nf = vals.shape[-1]
+ npts = pts.shape[1]
+ dev = pts.device
+ 
+ ind0 = torch.floor(pts / cubesize).long() # (batch, num_points, dim)
+ ind1 = torch.fmod(torch.ceil(pts / cubesize), size).long() # periodic wrap-around
+ ind01 = torch.stack((ind0, ind1), dim=0) # (2, batch, num_points, dim)
+ tmp = torch.tensor([0,1],dtype=torch.long)
+ com_ = torch.stack(torch.meshgrid(tuple([tmp] * dim)), dim=-1).view(-1, dim)
+ dim_ = torch.arange(dim).repeat(com_.shape[0], 1) # (2**dim, dim)
+ ind_ = ind01[com_, ..., dim_] # (2**dim, dim, batch, num_points)
+ ind_n = ind_.permute(2, 3, 0, 1) # (batch, num_points, 2**dim, dim)
+ # ind_b = torch.arange(bs).expand(ind_n.shape[1], ind_n.shape[2], bs).permute(2, 0, 1) # (batch, num_points, 2**dim)
+ ind_b = torch.arange(bs, device=dev).expand(ind_n.shape[1], ind_n.shape[2], bs).permute(2, 0, 1) # (batch, num_points, 2**dim)
+ 
+ # weights of neighboring nodes
+ xyz0 = ind0.type(cubesize.dtype) * cubesize # (batch, num_points, dim)
+ xyz1 = (ind0.type(cubesize.dtype) + 1) * cubesize # (batch, num_points, dim)
+ xyz01 = torch.stack((xyz0, xyz1), dim=0) # (2, batch, num_points, dim)
+ pos = xyz01[com_, ..., dim_].permute(2,3,0,1) # (batch, num_points, 2**dim, dim)
+ pos_ = xyz01[1-com_, ..., dim_].permute(2,3,0,1) # (batch, num_points, 2**dim, dim)
+ pos_ = pos_.type(pts.dtype)
+ dxyz_ = torch.abs(pts.unsqueeze(-2) - pos_) / cubesize # (batch, num_points, 2**dim, dim)
+ weights = torch.prod(dxyz_, dim=-1, keepdim=False) # (batch, num_points, 2**dim)
+ 
+ ind_b = ind_b.unsqueeze(-1).unsqueeze(-1) # (batch, num_points, 2**dim, 1, 1)
+ ind_n = ind_n.unsqueeze(-2) # (batch, num_points, 2**dim, 1, dim)
+ ind_f = torch.arange(nf, device=dev).view(1, 1, 1, nf, 1) # (1, 1, 1, nf, 1)
+ # ind_f = torch.arange(nf).view(1, 1, 1, nf, 1) # (1, 1, 1, nf, 1)
+ 
+ ind_b = ind_b.expand(bs, npts, 2**dim, nf, 1)
+ ind_n = ind_n.expand(bs, npts, 2**dim, nf, dim).to(dev)
+ ind_f = ind_f.expand(bs, npts, 2**dim, nf, 1)
+ inds = torch.cat([ind_b, ind_f, ind_n], dim=-1) # (batch, num_points, 2**dim, nf, 1+1+dim)
+ 
+ # weighted values
+ vals = weights.unsqueeze(-1) * vals.unsqueeze(-2) # (batch, num_points, 2**dim, nf)
+ 
+ inds = inds.view(-1, dim+2).permute(1, 0).long() # (1+dim+1, bs*npts*2**dim*nf)
+ vals = vals.reshape(-1) # (bs*npts*2**dim*nf)
+ tensor_size = [bs, nf] + size_list
+ raster = scatter_to_grid(inds.permute(1, 0), vals, [bs, nf] + size_list)
+ 
+ return raster # [batch, nf, res, res, res]
+
+
+
+##################################################
+# Below are the utilization functions in general
+
+class AverageMeter(object):
+ """Computes and stores the average and current value"""
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.val = 0
+ self.n = 0
+ self.avg = 0
+ self.sum = 0
+ self.count = 0
+
+ def update(self, val, n=1):
+ self.val = val
+ self.n = n
+ self.sum += val * n
+ self.count += n
+ self.avg = self.sum / self.count
+
+ @property
+ def valcavg(self):
+ return self.val.sum().item() / (self.n != 0).sum().item()
+
+ @property
+ def avgcavg(self):
+ return self.avg.sum().item() / (self.count != 0).sum().item()
+
+def load_model_manual(state_dict, model):
+ new_state_dict = OrderedDict()
+ is_model_parallel = isinstance(model, torch.nn.DataParallel)
+ for k, v in state_dict.items():
+ if k.startswith('module.') != is_model_parallel:
+ if k.startswith('module.'):
+ # remove module
+ k = k[7:]
+ else:
+ # add module
+ k = 'module.' + k
+
+ new_state_dict[k]=v
+
+ model.load_state_dict(new_state_dict)
+
+def mc_from_psr(psr_grid, pytorchify=False, real_scale=False, zero_level=0):
+ '''
+ Run marching cubes from PSR grid
+ '''
+ batch_size = psr_grid.shape[0]
+ s = psr_grid.shape[-1] # size of psr_grid
+ psr_grid_numpy = psr_grid.squeeze().detach().cpu().numpy()
+ 
+ if batch_size>1:
+ verts, faces, normals = [], [], []
+ for i in range(batch_size):
+ verts_cur, faces_cur, normals_cur, values = measure.marching_cubes(psr_grid_numpy[i], level=0)
+ verts.append(verts_cur)
+ faces.append(faces_cur)
+ normals.append(normals_cur)
+ verts = np.stack(verts, axis = 0)
+ faces = np.stack(faces, axis = 0)
+ normals = np.stack(normals, axis = 0)
+ else:
+ try:
+ verts, faces, normals, values = measure.marching_cubes(psr_grid_numpy, level=zero_level)
+ except:
+ verts, faces, normals, values = measure.marching_cubes(psr_grid_numpy)
+ if real_scale:
+ verts = verts / (s-1) # scale to range [0, 1]
+ else:
+ verts = verts / s # scale to range [0, 1)
+
+ if pytorchify:
+ device = psr_grid.device
+ verts = torch.Tensor(np.ascontiguousarray(verts)).to(device)
+ faces = torch.Tensor(np.ascontiguousarray(faces)).to(device)
+ normals = torch.Tensor(np.ascontiguousarray(-normals)).to(device)
+
+ return verts, faces, normals
+ 
+def calc_inters_points(verts, faces, pose, img_size, mask_gt=None):
+ verts = verts.squeeze()
+ faces = faces.squeeze()
+ pix_to_face, w, mask = mesh_rasterization(verts, faces, pose, img_size)
+ if mask_gt is not None:
+ #! only evaluate within the intersection
+ mask = mask & mask_gt
+ # find 3D points intesected on the mesh
+ if True:
+ w_masked = w[mask]
+ f_p = faces[pix_to_face[mask]].long() # cooresponding faces for each pixel
+ # corresponding vertices for p_closest
+ v_a, v_b, v_c = verts[f_p[..., 0]], verts[f_p[..., 1]], verts[f_p[..., 2]]
+ 
+ # calculate the intersection point of each pixel and the mesh
+ p_inters = w_masked[..., 0, None] * v_a + \
+ w_masked[..., 1, None] * v_b + \
+ w_masked[..., 2, None] * v_c
+ else:
+ # backproject ndc to world coordinates using z-buffer
+ W, H = img_size[1], img_size[0]
+ xy = uv.to(mask.device)[mask]
+ x_ndc = 1 - (2*xy[:, 0]) / (W - 1)
+ y_ndc = 1 - (2*xy[:, 1]) / (H - 1)
+ z = zbuf.squeeze().reshape(H * W)[mask]
+ xy_depth = torch.stack((x_ndc, y_ndc, z), dim=1)
+ 
+ p_inters = pose.unproject_points(xy_depth, world_coordinates=True)
+ 
+ # if there are outlier points, we should remove it
+ if (p_inters.max()>1) | (p_inters.min()<-1):
+ mask_bound = (p_inters>=-1) & (p_inters<=1)
+ mask_bound = (mask_bound.sum(dim=-1)==3)
+ mask[mask==True] = mask_bound
+ p_inters = p_inters[mask_bound]
+ print('!!!!!find outlier!')
+
+ return p_inters, mask, f_p, w_masked
+
+def mesh_rasterization(verts, faces, pose, img_size):
+ '''
+ Use PyTorch3D to rasterize the mesh given a camera 
+ '''
+ transformed_v = pose.transform_points(verts.detach()) # world -> ndc coordinate system
+ if isinstance(pose, PerspectiveCameras):
+ transformed_v[..., 2] = 1/transformed_v[..., 2]
+ # find p_closest on mesh of each pixel via rasterization
+ transformed_mesh = Meshes(verts=[transformed_v], faces=[faces])
+ pix_to_face, zbuf, bary_coords, dists = rasterize_meshes(
+ transformed_mesh,
+ image_size=img_size,
+ blur_radius=0,
+ faces_per_pixel=1,
+ perspective_correct=False
+ )
+ pix_to_face = pix_to_face.reshape(1, -1) # B x reso x reso -> B x (reso x reso)
+ mask = pix_to_face.clone() != -1
+ mask = mask.squeeze()
+ pix_to_face = pix_to_face.squeeze()
+ w = bary_coords.reshape(-1, 3)
+
+ return pix_to_face, w, mask
+
+def verts_on_largest_mesh(verts, faces):
+ '''
+ verts: Numpy array or Torch.Tensor (N, 3)
+ faces: Numpy array (N, 3)
+ '''
+ if torch.is_tensor(faces):
+ verts = verts.squeeze().detach().cpu().numpy()
+ faces = faces.squeeze().int().detach().cpu().numpy()
+
+ A = adjacency_matrix(faces)
+ num, conn_idx, conn_size = connected_components(A)
+ if num == 0:
+ v_large, f_large = verts, faces
+ else:
+ max_idx = conn_size.argmax() # find the index of the largest component
+ v_large = verts[conn_idx==max_idx] # keep points on the largest component
+
+ if True:
+ mesh_largest = trimesh.Trimesh(verts, faces)
+ connected_comp = mesh_largest.split(only_watertight=False)
+ mesh_largest = connected_comp[max_idx]
+ v_large, f_large = mesh_largest.vertices, mesh_largest.faces
+ v_large = v_large.astype(np.float32)
+ return v_large, f_large
+
+def update_recursive(dict1, dict2):
+ ''' Update two config dictionaries recursively.
+
+ Args:
+ dict1 (dict): first dictionary to be updated
+ dict2 (dict): second dictionary which entries should be used
+
+ '''
+ for k, v in dict2.items():
+ if k not in dict1:
+ dict1[k] = dict()
+ if isinstance(v, dict):
+ update_recursive(dict1[k], v)
+ else:
+ dict1[k] = v
+
+def scale2onet(p, scale=1.2):
+ '''
+ Scale the point cloud from SAP to ONet range
+ '''
+ return (p - 0.5) * scale
+ 
+def update_optimizer(inputs, cfg, epoch, model=None, schedule=None):
+ if model is not None:
+ if schedule is not None:
+ optimizer = torch.optim.Adam([
+ {"params": model.parameters(),
+ "lr": schedule[0].get_learning_rate(epoch)},
+ {"params": inputs,
+ "lr": schedule[1].get_learning_rate(epoch)}])
+ elif 'lr' in cfg['train']:
+ optimizer = torch.optim.Adam([
+ {"params": model.parameters(),
+ "lr": float(cfg['train']['lr'])},
+ {"params": inputs,
+ "lr": float(cfg['train']['lr_pcl'])}])
+ else:
+ raise Exception('no known learning rate')
+ else:
+ if schedule is not None:
+ optimizer = torch.optim.Adam([inputs], lr=schedule[0].get_learning_rate(epoch))
+ else:
+ optimizer = torch.optim.Adam([inputs], lr=float(cfg['train']['lr_pcl']))
+
+ return optimizer
+
+
+def is_url(url):
+ scheme = urllib.parse.urlparse(url).scheme
+ return scheme in ('http', 'https')
+
+def load_url(url):
+ '''Load a module dictionary from url.
+ 
+ Args:
+ url (str): url to saved model
+ '''
+ print(url)
+ print('=> Loading checkpoint from url...')
+ state_dict = model_zoo.load_url(url, progress=True)
+ 
+ return state_dict
+
+
+class GaussianSmoothing(nn.Module):
+ """
+ Apply gaussian smoothing on a
+ 1d, 2d or 3d tensor. Filtering is performed seperately for each channel
+ in the input using a depthwise convolution.
+ Arguments:
+ channels (int, sequence): Number of channels of the input tensors. Output will have this number of channels as well.
+ kernel_size (int, sequence): Size of the gaussian kernel.
+ sigma (float, sequence): Standard deviation of the gaussian kernel.
+ dim (int, optional): The number of dimensions of the data.
+ Default value is 2 (spatial).
+ """
+ def __init__(self, channels, kernel_size, sigma, dim=3):
+ super(GaussianSmoothing, self).__init__()
+ if isinstance(kernel_size, numbers.Number):
+ kernel_size = [kernel_size] * dim
+ if isinstance(sigma, numbers.Number):
+ sigma = [sigma] * dim
+
+ # The gaussian kernel is the product of the
+ # gaussian function of each dimension.
+ kernel = 1
+ meshgrids = torch.meshgrid(
+ [
+ torch.arange(size, dtype=torch.float32)
+ for size in kernel_size
+ ]
+ )
+ for size, std, mgrid in zip(kernel_size, sigma, meshgrids):
+ mean = (size - 1) / 2
+ kernel *= 1 / (std * math.sqrt(2 * math.pi)) * \
+ torch.exp(-((mgrid - mean) / std) ** 2 / 2)
+
+ # Make sure sum of values in gaussian kernel equals 1.
+ kernel = kernel / torch.sum(kernel)
+
+ # Reshape to depthwise convolutional weight
+ kernel = kernel.view(1, 1, *kernel.size())
+ kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1))
+
+ self.register_buffer('weight', kernel)
+ self.groups = channels
+
+ if dim == 1:
+ self.conv = F.conv1d
+ elif dim == 2:
+ self.conv = F.conv2d
+ elif dim == 3:
+ self.conv = F.conv3d
+ else:
+ raise RuntimeError(
+ 'Only 1, 2 and 3 dimensions are supported. Received {}.'.format(dim)
+ )
+
+ def forward(self, input):
+ """
+ Apply gaussian filter to input.
+ Arguments:
+ input (torch.Tensor): Input to apply gaussian filter on.
+ Returns:
+ filtered (torch.Tensor): Filtered output.
+ """
+ return self.conv(input, weight=self.weight, groups=self.groups)
+ 
+# Originally from https://github.com/amosgropp/IGR/blob/0db06b1273/code/utils/general.py
+def get_learning_rate_schedules(schedule_specs):
+
+ schedules = []
+
+ for key in schedule_specs.keys():
+ schedules.append(StepLearningRateSchedule(
+ schedule_specs[key]['initial'],
+ schedule_specs[key]["interval"],
+ schedule_specs[key]["factor"], 
+ schedule_specs[key]["final"]))
+ return schedules
+
+class LearningRateSchedule:
+ def get_learning_rate(self, epoch):
+ pass
+class StepLearningRateSchedule(LearningRateSchedule):
+ def __init__(self, initial, interval, factor, final=1e-6):
+ self.initial = float(initial)
+ self.interval = interval
+ self.factor = factor
+ self.final = float(final)
+
+ def get_learning_rate(self, epoch):
+ lr = np.maximum(self.initial * (self.factor ** (epoch // self.interval)), 5.0e-6)
+ if lr > self.final:
+ return lr
+ else:
+ return self.final
+
+def adjust_learning_rate(lr_schedules, optimizer, epoch):
+ for i, param_group in enumerate(optimizer.param_groups):
+ param_group["lr"] = lr_schedules[i].get_learning_rate(epoch)

models/model.py

@@ -0,0 +1,181 @@
+import random
+from statistics import mean
+from typing import List, Tuple
+
+import torch as th
+import pytorch_lightning as pl
+from jaxtyping import Float, Int
+import numpy as np
+from torch_geometric.nn.conv import GATv2Conv
+
+from models.SAP.dpsr import DPSR
+from models.SAP.model import PSR2Mesh
+
+# Constants
+
+th.manual_seed(0)
+np.random.seed(0)
+
+BATCH_SIZE = 1 # BS
+
+IN_DIM = 1 
+OUT_DIM = 1
+LATENT_DIM = 32
+
+DROPOUT_PROB = 0.1
+GRID_SIZE = 128
+
+def generate_grid_edge_list(gs: int = 128):
+ grid_edge_list = []
+
+ for k in range(gs):
+ for j in range(gs):
+ for i in range(gs):
+ current_idx = i + gs*j + k*gs*gs
+ if (i - 1) >= 0:
+ grid_edge_list.append([current_idx, i-1 + gs*j + k*gs*gs])
+ if (i + 1) < gs:
+ grid_edge_list.append([current_idx, i+1 + gs*j + k*gs*gs])
+ if (j - 1) >= 0:
+ grid_edge_list.append([current_idx, i + gs*(j-1) + k*gs*gs])
+ if (j + 1) < gs:
+ grid_edge_list.append([current_idx, i + gs*(j+1) + k*gs*gs])
+ if (k - 1) >= 0:
+ grid_edge_list.append([current_idx, i + gs*j + (k-1)*gs*gs])
+ if (k + 1) < gs:
+ grid_edge_list.append([current_idx, i + gs*j + (k+1)*gs*gs])
+ return grid_edge_list
+
+GRID_EDGE_LIST = generate_grid_edge_list(GRID_SIZE)
+GRID_EDGE_LIST = th.tensor(GRID_EDGE_LIST, dtype=th.int)
+GRID_EDGE_LIST = GRID_EDGE_LIST.T
+# GRID_EDGE_LIST = GRID_EDGE_LIST.to(th.device("cuda"))
+GRID_EDGE_LIST.requires_grad = False # Do not forget to delete it if train
+
+
+class FormOptimizer(th.nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ 
+ layers = []
+ 
+ self.gconv1 = GATv2Conv(in_channels=IN_DIM, out_channels=LATENT_DIM, heads=1, dropout=DROPOUT_PROB)
+ self.gconv2 = GATv2Conv(in_channels=LATENT_DIM, out_channels=LATENT_DIM, heads=1, dropout=DROPOUT_PROB)
+ 
+ self.actv = th.nn.Sigmoid()
+ self.head = th.nn.Linear(in_features=LATENT_DIM, out_features=OUT_DIM)
+
+ def forward(self, 
+ field: Float[th.Tensor, "GS GS GS"]) -> Float[th.Tensor, "GS GS GS"]:
+ """
+ Args:
+ field (Tensor [GS, GS, GS]): vertices and normals tensor.
+ """
+ vertex_features = field.clone()
+ vertex_features = vertex_features.reshape(GRID_SIZE*GRID_SIZE*GRID_SIZE, IN_DIM)
+ 
+ vertex_features = self.gconv1(x=vertex_features, edge_index=GRID_EDGE_LIST) 
+ vertex_features = self.gconv2(x=vertex_features, edge_index=GRID_EDGE_LIST) 
+ field_delta = self.head(self.actv(vertex_features))
+ 
+ field_delta = field_delta.reshape(BATCH_SIZE, GRID_SIZE, GRID_SIZE, GRID_SIZE)
+ field_delta += field # field_delta carries the gradient
+ field_delta = th.clamp(field_delta, min=-0.5, max=0.5)
+ 
+ return field_delta
+
+class Model(pl.LightningModule):
+ def __init__(self):
+ super().__init__()
+ self.form_optimizer = FormOptimizer()
+ 
+ self.dpsr = DPSR([GRID_SIZE, GRID_SIZE, GRID_SIZE], sig=0.0)
+ self.field2mesh = PSR2Mesh().apply
+
+ self.metric = th.nn.MSELoss()
+ 
+ self.val_losses = []
+ self.train_losses = []
+
+ def log_h5(self, points, normals):
+ dset = self.log_points_file.create_dataset(
+ name=str(self.h5_frame),
+ shape=points.shape,
+ dtype=np.float16, 
+ compression="gzip")
+ dset[:] = points
+ dset = self.log_normals_file.create_dataset(
+ name=str(self.h5_frame),
+ shape=normals.shape,
+ dtype=np.float16, 
+ compression="gzip")
+ dset[:] = normals
+ self.h5_frame += 1
+ 
+ def forward(self, 
+ v: Float[th.Tensor, "BS N 3"],
+ n: Float[th.Tensor, "BS N 3"]) -> Tuple[Float[th.Tensor, "BS N 3"], # v - vertices
+ Int[th.Tensor, "2 E"], # f - faces
+ Float[th.Tensor, "BS N 3"], # n - vertices normals
+ Float[th.Tensor, "BS GR GR GR"]]: # field: 
+ field = self.dpsr(v, n)
+ field = self.form_optimizer(field)
+ v, f, n = self.field2mesh(field)
+ return v, f, n, field
+
+ def training_step(self, batch, batch_idx) -> Float[th.Tensor, "1"]:
+ vertices, vertices_normals, vertices_gt, vertices_normals_gt, field_gt, adj = batch
+ 
+ mask = th.rand((vertices.shape[1], ), device=th.device("cuda")) < (random.random() / 2.0 + 0.5)
+ vertices = vertices[:, mask]
+ vertices_normals = vertices_normals[:, mask]
+ 
+ vr, fr, nr, field_r = model(vertices, vertices_normals)
+ 
+ loss = self.metric(field_r, field_gt)
+ train_per_step_loss = loss.item()
+ self.train_losses.append(train_per_step_loss)
+ 
+ return loss
+ 
+ def on_train_epoch_end(self):
+ mean_train_per_epoch_loss = mean(self.train_losses)
+ self.log("mean_train_per_epoch_loss", mean_train_per_epoch_loss, on_step=False, on_epoch=True)
+ self.train_losses = []
+ 
+ def validation_step(self, batch, batch_idx):
+ vertices, vertices_normals, vertices_gt, vertices_normals_gt, field_gt, adj = batch
+ 
+ vr, fr, nr, field_r = model(vertices, vertices_normals)
+ 
+ loss = self.metric(field_r, field_gt)
+ val_per_step_loss = loss.item()
+ self.val_losses.append(val_per_step_loss)
+ return loss
+ 
+ def on_validation_epoch_end(self):
+ mean_val_per_epoch_loss = mean(self.val_losses)
+ self.log("mean_val_per_epoch_loss", mean_val_per_epoch_loss, on_step=False, on_epoch=True)
+ self.val_losses = []
+
+ def configure_optimizers(self):
+ optimizer = th.optim.Adam(self.parameters(), lr=LR)
+ scheduler = th.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5)
+ 
+ return {
+ "optimizer": optimizer,
+ "lr_scheduler": {
+ "scheduler": scheduler, 
+ "monitor": "mean_val_per_epoch_loss",
+ "interval": "epoch",
+ "frequency": 1,
+ # If set to `True`, will enforce that the value specified 'monitor'
+ # is available when the scheduler is updated, thus stopping
+ # training if not found. If set to `False`, it will only produce a warning
+ "strict": True,
+ # If using the `LearningRateMonitor` callback to monitor the
+ # learning rate progress, this keyword can be used to specify
+ # a custom logged name
+ "name": None,
+ }
+ }

requirements.txt

@@ -0,0 +1,298 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+ansicon==1.89.0
+anyio==3.6.2
+arrow==1.2.3
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+beautifulsoup4==4.12.2
+blessed==1.20.0
+blinker==1.6.2
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+colorama==0.4.6
+comm==0.1.3
+ConfigArgParse==1.5.3
+croniter==1.3.14
+dash==2.9.3
+dash-core-components==2.0.0
+dash-html-components==2.0.0
+dash-table==5.0.0
+dateutils==0.6.12
+debugpy==1.6.7
+decorator==5.1.1
+deepdiff==6.3.0
+executing==1.2.0
+fastapi==0.88.0
+fastjsonschema==2.16.3
+Flask==2.3.2
+frozenlist==1.3.3
+fsspec==2023.5.0
+fvcore==0.1.5.post20221221
+h11==0.14.0
+idna==3.4
+imageio==2.28.1
+importlib-metadata==6.6.0
+inquirer==3.1.3
+iopath==0.1.10
+ipykernel==6.23.1
+ipython==8.13.2
+ipywidgets==8.0.6
+itsdangerous==2.1.2
+jaxtyping==0.2.19
+jedi==0.18.2
+Jinja2==3.1.2
+jinxed==1.2.0
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter_client==8.2.0
+jupyter_core==5.3.0
+jupyterlab-widgets==3.0.7
+lazy_loader==0.2
+libigl==2.4.1
+lightning==2.0.2
+lightning-cloud==0.5.36
+lightning-utilities==0.8.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+multidict==6.0.4
+nbformat==5.7.0
+nest-asyncio==1.5.6
+networkx==3.1
+numpy==1.24.3
+open3d==0.17.0
+ordered-set==4.1.0
+packaging==23.1
+parso==0.8.3
+pickleshare==0.7.5
+Pillow==9.5.0
+platformdirs==3.5.1
+plotly==5.14.1
+plyfile==0.9
+portalocker==2.7.0
+prompt-toolkit==3.0.38
+psutil==5.9.5
+pure-eval==0.2.2
+pydantic==1.10.7
+Pygments==2.15.1
+PyJWT==2.7.0
+pyparsing==3.0.9
+pyrsistent==0.19.3
+PySimpleGUI==4.60.4
+python-dateutil==2.8.2
+python-editor==1.0.4
+python-multipart==0.0.6
+pytorch-lightning==2.0.2
+pytz==2023.3
+PyWavelets==1.4.1
+pywin32==306
+PyYAML==6.0
+pyzmq==25.0.2
+readchar==4.0.5
+requests==2.30.0
+rich==13.3.5
+scikit-image==0.20.0
+scikit-learn==1.2.2
+scipy==1.9.1
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+stack-data==0.6.2
+starlette==0.22.0
+starsessions==1.3.0
+tabulate==0.9.0
+tenacity==8.2.2
+termcolor==2.3.0
+threadpoolctl==3.1.0
+tifffile==2023.4.12
+torch==1.13.1+cu116
+torch-cluster==1.6.1+pt113cu116
+torch-geometric==2.3.1
+torch-scatter==2.1.1+pt113cu116
+torch-sparse==0.6.17+pt113cu116
+torch-spline-conv==1.2.2+pt113cu116
+torchaudio==0.13.1
+torchmetrics==0.11.4
+torchvision==0.14.1+cu116
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+trimesh==3.21.6
+typeguard==4.0.0
+typing_extensions==4.5.0
+urllib3==2.0.2
+uvicorn==0.22.0
+wcwidth==0.2.6
+websocket-client==1.5.1
+websockets==11.0.3
+Werkzeug==2.3.4
+widgetsnbextension==4.0.7
+yacs==0.1.8
+yarl==1.9.2
+zipp==3.15.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.0
+ansicon==1.89.0
+anyio==3.6.2
+arrow==1.2.3
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+beautifulsoup4==4.12.2
+blessed==1.20.0
+blinker==1.6.2
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+colorama==0.4.6
+comm==0.1.3
+ConfigArgParse==1.5.3
+contourpy==1.0.7
+croniter==1.3.14
+cycler==0.11.0
+dash==2.9.3
+dash-core-components==2.0.0
+dash-html-components==2.0.0
+dash-table==5.0.0
+dateutils==0.6.12
+debugpy==1.6.7
+decorator==5.1.1
+deepdiff==6.3.0
+executing==1.2.0
+fastapi==0.88.0
+fastjsonschema==2.16.3
+ffmpy==0.3.0
+filelock==3.12.0
+Flask==2.3.2
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+fvcore==0.1.5.post20221221
+gradio==3.30.0
+gradio_client==0.2.5
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.14.1
+idna==3.4
+imageio==2.28.1
+importlib-metadata==6.6.0
+importlib-resources==5.12.0
+inquirer==3.1.3
+iopath==0.1.10
+ipykernel==6.23.1
+ipython==8.13.2
+ipywidgets==8.0.6
+itsdangerous==2.1.2
+jaxtyping==0.2.19
+jedi==0.18.2
+Jinja2==3.1.2
+jinxed==1.2.0
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter_client==8.2.0
+jupyter_core==5.3.0
+jupyterlab-widgets==3.0.7
+kiwisolver==1.4.4
+lazy_loader==0.2
+libigl==2.4.1
+lightning==2.0.2
+lightning-cloud==0.5.36
+lightning-utilities==0.8.0
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+nbformat==5.7.0
+nest-asyncio==1.5.6
+networkx==3.1
+numpy==1.24.3
+open3d==0.17.0
+ordered-set==4.1.0
+orjson==3.8.12
+packaging==23.1
+pandas==2.0.1
+parso==0.8.3
+pickleshare==0.7.5
+Pillow==9.5.0
+platformdirs==3.5.1
+plotly==5.14.1
+plyfile==0.9
+portalocker==2.7.0
+prompt-toolkit==3.0.38
+psutil==5.9.5
+pure-eval==0.2.2
+pydantic==1.10.7
+pydub==0.25.1
+Pygments==2.15.1
+PyJWT==2.7.0
+pyparsing==3.0.9
+pyrsistent==0.19.3
+PySimpleGUI==4.60.4
+python-dateutil==2.8.2
+python-editor==1.0.4
+python-multipart==0.0.6
+pytorch-lightning==2.0.2
+pytz==2023.3
+PyWavelets==1.4.1
+pywin32==306
+PyYAML==6.0
+pyzmq==25.0.2
+readchar==4.0.5
+requests==2.30.0
+rich==13.3.5
+scikit-image==0.20.0
+scikit-learn==1.2.2
+scipy==1.9.1
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+stack-data==0.6.2
+starlette==0.22.0
+starsessions==1.3.0
+tabulate==0.9.0
+tenacity==8.2.2
+termcolor==2.3.0
+threadpoolctl==3.1.0
+tifffile==2023.4.12
+toolz==0.12.0
+torch==1.13.1+cu116
+torch-cluster==1.6.1+pt113cu116
+torch-geometric==2.3.1
+torch-scatter==2.1.1+pt113cu116
+torch-sparse==0.6.17+pt113cu116
+torch-spline-conv==1.2.2+pt113cu116
+torchaudio==0.13.1
+torchmetrics==0.11.4
+torchvision==0.14.1+cu116
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+trimesh==3.21.6
+typeguard==4.0.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==2.0.2
+uvicorn==0.22.0
+wcwidth==0.2.6
+websocket-client==1.5.1
+websockets==11.0.3
+Werkzeug==2.3.4
+widgetsnbextension==4.0.7
+yacs==0.1.8
+yarl==1.9.2
+zipp==3.15.0