import io import cv2 import numpy as np from PIL import Image from skimage.transform import resize import matplotlib.pyplot as plt from mpl_toolkits.mplot3d import Axes3D def draw_hand3d(keypoints): # Define the connections between keypoints as tuples (start, end) bones = [ ((0, 1), 'red'), ((1, 2), 'green'), ((2, 3), 'blue'), ((3, 4), 'purple'), ((0, 5), 'orange'), ((5, 6), 'pink'), ((6, 7), 'brown'), ((7, 8), 'cyan'), ((0, 9), 'yellow'), ((9, 10), 'magenta'), ((10, 11), 'lime'), ((11, 12), 'blueviolet'), ((0, 13), 'olive'), ((13, 14), 'teal'), ((14, 15), 'crimson'), ((15, 16), 'cornsilk'), ((0, 17), 'aqua'), ((17, 18), 'silver'), ((18, 19), 'maroon'), ((19, 20), 'fuchsia') ] fig = plt.figure() ax = fig.add_subplot(111, projection='3d') # Plot the bones for bone, color in bones: start_point = keypoints[bone[0], :] end_point = keypoints[bone[1], :] ax.plot([start_point[0], end_point[0]], [start_point[1], end_point[1]], [start_point[2], end_point[2]], color=color) ax.scatter(keypoints[:, 0], keypoints[:, 1], keypoints[:, 2], color='gray', s=15) # Set the aspect ratio to be equal max_range = np.array([keypoints[:,0].max()-keypoints[:,0].min(), keypoints[:,1].max()-keypoints[:,1].min(), keypoints[:,2].max()-keypoints[:,2].min()]).max() / 2.0 mid_x = (keypoints[:,0].max()+keypoints[:,0].min()) * 0.5 mid_y = (keypoints[:,1].max()+keypoints[:,1].min()) * 0.5 mid_z = (keypoints[:,2].max()+keypoints[:,2].min()) * 0.5 ax.set_xlim(mid_x - max_range, mid_x + max_range) ax.set_ylim(mid_y - max_range, mid_y + max_range) ax.set_zlim(mid_z - max_range, mid_z + max_range) # Set labels for axes ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') plt.show() def visualize_hand(joints, img): # Define the connections between joints for drawing lines and their corresponding colors connections = [ ((0, 1), 'red'), ((1, 2), 'green'), ((2, 3), 'blue'), ((3, 4), 'purple'), ((0, 5), 'orange'), ((5, 6), 'pink'), ((6, 7), 'brown'), ((7, 8), 'cyan'), ((0, 9), 'yellow'), ((9, 10), 'magenta'), ((10, 11), 'lime'), ((11, 12), 'indigo'), ((0, 13), 'olive'), ((13, 14), 'teal'), ((14, 15), 'navy'), ((15, 16), 'gray'), ((0, 17), 'lavender'), ((17, 18), 'silver'), ((18, 19), 'maroon'), ((19, 20), 'fuchsia') ] H, W, C = img.shape # Create a figure and axis plt.figure() ax = plt.gca() # Plot joints as points ax.imshow(img) ax.scatter(joints[:, 0], joints[:, 1], color='white', s=15) # Plot lines connecting joints with different colors for each bone for connection, color in connections: joint1 = joints[connection[0]] joint2 = joints[connection[1]] ax.plot([joint1[0], joint2[0]], [joint1[1], joint2[1]], color=color) ax.set_xlim([0, W]) ax.set_ylim([0, H]) ax.grid(False) ax.set_axis_off() ax.invert_yaxis() plt.subplots_adjust(wspace=0.01) plt.show() def draw_hand_skeleton(joints, image_size, thickness=5): # Create a blank white image image = np.zeros((image_size[0], image_size[1]), dtype=np.uint8) # Define the connections between joints connections = [ (0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (0, 9), (9, 10), (10, 11), (11, 12), (0, 13), (13, 14), (14, 15), (15, 16), (0, 17), (17, 18), (18, 19), (19, 20), ] # Draw lines connecting joints for connection in connections: joint1 = joints[connection[0]].astype("int") joint2 = joints[connection[1]].astype("int") cv2.line(image, tuple(joint1), tuple(joint2), color=1, thickness=thickness) return image def draw_hand(joints, img): # Define the connections between joints for drawing lines and their corresponding colors connections = [ ((0, 1), 'red'), ((1, 2), 'green'), ((2, 3), 'blue'), ((3, 4), 'purple'), ((0, 5), 'orange'), ((5, 6), 'pink'), ((6, 7), 'brown'), ((7, 8), 'cyan'), ((0, 9), 'yellow'), ((9, 10), 'magenta'), ((10, 11), 'lime'), ((11, 12), 'indigo'), ((0, 13), 'olive'), ((13, 14), 'teal'), ((14, 15), 'navy'), ((15, 16), 'gray'), ((0, 17), 'lavender'), ((17, 18), 'silver'), ((18, 19), 'maroon'), ((19, 20), 'fuchsia') ] H, W, C = img.shape # Create a figure and axis with the same size as the input image fig, ax = plt.subplots(figsize=(W / 100, H / 100), dpi=100) # Plot joints as points ax.imshow(img) ax.scatter(joints[:, 0], joints[:, 1], color='white', s=15) # Plot lines connecting joints with different colors for each bone for connection, color in connections: joint1 = joints[connection[0]] joint2 = joints[connection[1]] ax.plot([joint1[0], joint2[0]], [joint1[1], joint2[1]], color=color) ax.set_xlim([0, W]) ax.set_ylim([0, H]) ax.grid(False) ax.set_axis_off() ax.invert_yaxis() plt.subplots_adjust(left=0, right=1, top=1, bottom=0, wspace=0.01, hspace=0.01) # Save the plot to a buffer buf = io.BytesIO() plt.savefig(buf, format='png', bbox_inches='tight', pad_inches=0) plt.close(fig) # Close the figure to free memory # Load the image from the buffer into a PIL image and then into a numpy array buf.seek(0) img_arr = np.array(Image.open(buf)) return img_arr[..., :3] def keypoint_heatmap(pts, size, var=1.0): H, W = size x = np.linspace(0, W - 1, W) y = np.linspace(0, H - 1, H) xv, yv = np.meshgrid(x, y) grid = np.stack((xv, yv), axis=-1) # Expanding dims for broadcasting subtraction between pts and every grid position modes_exp = np.expand_dims(np.expand_dims(pts, axis=1), axis=1) # Calculating squared difference diff = grid - modes_exp normal = np.exp(-np.sum(diff**2, axis=-1) / (2 * var)) / ( 2.0 * np.pi * var ) return normal def check_keypoints_validity(keypoints, image_size): H, W = image_size # Check if x coordinates are valid: 0 < x < W valid_x = (keypoints[:, 0] > 0) & (keypoints[:, 0] < W) # Check if y coordinates are valid: 0 < y < H valid_y = (keypoints[:, 1] > 0) & (keypoints[:, 1] < H) # Combine the validity checks for both x and y valid_keypoints = valid_x & valid_y # Convert boolean array to integer (1 for True, 0 for False) return valid_keypoints.astype(int) def find_bounding_box(mask, margin=30): """Find the bounding box of a binary mask. Return None if the mask is empty.""" rows = np.any(mask, axis=1) cols = np.any(mask, axis=0) if not rows.any() or not cols.any(): # Mask is empty return None ymin, ymax = np.where(rows)[0][[0, -1]] xmin, xmax = np.where(cols)[0][[0, -1]] xmin -= margin xmax += margin ymin -= margin ymax += margin return xmin, ymin, xmax, ymax def adjust_box_to_image(xmin, ymin, xmax, ymax, image_width, image_height): """Adjust the bounding box to fit within the image boundaries.""" box_width = xmax - xmin box_height = ymax - ymin # Determine the side length of the square (the larger of the two dimensions) side_length = max(box_width, box_height) # Adjust to maintain a square by expanding or contracting sides xmin = max(0, xmin - (side_length - box_width) // 2) xmax = xmin + side_length ymin = max(0, ymin - (side_length - box_height) // 2) ymax = ymin + side_length # Ensure the box is still within the image boundaries after adjustments if xmax > image_width: shift = xmax - image_width xmin -= shift xmax -= shift if ymax > image_height: shift = ymax - image_height ymin -= shift ymax -= shift # After shifting, double-check if any side is out-of-bounds and adjust if necessary xmin = max(0, xmin) ymin = max(0, ymin) xmax = min(image_width, xmax) ymax = min(image_height, ymax) # It's possible the adjustments made the box not square (due to boundary constraints), # so we might need to slightly adjust the size to keep it as square as possible # This could involve a final adjustment based on the specific requirements, # like reducing the side length to fit or deciding which dimension to prioritize. return xmin, ymin, xmax, ymax def scale_keypoint(keypoint, original_size, target_size): """Scale a keypoint based on the resizing of the image.""" keypoint_copy = keypoint.copy() keypoint_copy[:, 0] *= target_size[0] / original_size[0] keypoint_copy[:, 1] *= target_size[1] / original_size[1] return keypoint_copy def crop_and_adjust_image_and_annotations(image, hand_mask, obj_mask, hand_pose, intrinsics, target_size=(512, 512)): # Find bounding boxes for each mask, handling potentially empty masks xmin, ymin, xmax, ymax = find_bounding_box(hand_mask) if np.any(hand_mask) else None # Adjust bounding box to fit within the image and be square xmin, ymin, xmax, ymax = adjust_box_to_image(xmin, ymin, xmax, ymax, image.shape[1], image.shape[0]) # Crop the image and mask # masked_hand_image = (image * np.maximum(hand_mask, obj_mask)[..., None].astype(float)).astype(np.uint8) cropped_hand_image = image[ymin:ymax, xmin:xmax] cropped_hand_mask = hand_mask[ymin:ymax, xmin:xmax].astype(np.uint8) cropped_obj_mask = obj_mask[ymin:ymax, xmin:xmax].astype(np.uint8) # Resize the image resized_image = resize(cropped_hand_image, target_size, anti_aliasing=True) resized_hand_mask = cv2.resize(cropped_hand_mask, dsize=target_size, interpolation=cv2.INTER_NEAREST) resized_obj_mask = cv2.resize(cropped_obj_mask, dsize=target_size, interpolation=cv2.INTER_NEAREST) # adjust and scale 2d keypoints for hand_type, kps2d in hand_pose.items(): kps2d[:, 0] -= xmin kps2d[:, 1] -= ymin hand_pose[hand_type] = scale_keypoint(kps2d, (xmax - xmin, ymax - ymin), target_size) # adjust instrinsics resized_intrinsics= np.array(intrinsics, copy=True) resized_intrinsics[0, 2] -= xmin resized_intrinsics[1, 2] -= ymin resized_intrinsics[0, :] *= target_size[0] / (xmax - xmin) resized_intrinsics[1, :] *= target_size[1] / (ymax - ymin) return (resized_image, resized_hand_mask, resized_obj_mask, hand_pose, resized_intrinsics)