File size: 5,446 Bytes
7384b30 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 |
---
library_name: transformers.js
tags:
- pose-estimation
license: apache-2.0
---
https://github.com/open-mmlab/mmpose/tree/main/projects/rtmo with ONNX weights to be compatible with Transformers.js.
## Usage (Transformers.js)
If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using:
```bash
npm i @xenova/transformers
```
**Example:** Perform pose-estimation w/ `Xenova/RTMO-s`.
```js
import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers';
// Load model and processor
const model_id = 'Xenova/RTMO-s';
const model = await AutoModel.from_pretrained(model_id);
const processor = await AutoProcessor.from_pretrained(model_id);
// Read image and run processor
const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg';
const image = await RawImage.read(url);
const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image);
// Predict bounding boxes and keypoints
const { dets, keypoints } = await model({ input: pixel_values });
// Select the first image
const predicted_boxes = dets.tolist()[0];
const predicted_points = keypoints.tolist()[0];
const [height, width] = original_sizes[0];
const [resized_height, resized_width] = reshaped_input_sizes[0];
// Compute scale values
const xScale = width / resized_width;
const yScale = height / resized_height;
// Define thresholds
const point_threshold = 0.3;
const box_threshold = 0.3;
// Display results
for (let i = 0; i < predicted_boxes.length; ++i) {
const [xmin, ymin, xmax, ymax, box_score] = predicted_boxes[i];
if (box_score < box_threshold) continue;
const x1 = (xmin * xScale).toFixed(2);
const y1 = (ymin * yScale).toFixed(2);
const x2 = (xmax * xScale).toFixed(2);
const y2 = (ymax * yScale).toFixed(2);
console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${box_score.toFixed(3)}`)
const points = predicted_points[i]; // of shape [17, 3]
for (let id = 0; id < points.length; ++id) {
const label = model.config.id2label[id];
const [x, y, point_score] = points[id];
if (point_score < point_threshold) continue;
console.log(` - ${label}: (${(x * xScale).toFixed(2)}, ${(y * yScale).toFixed(2)}) with score ${point_score.toFixed(3)}`);
}
}
```
<details>
<summary>See example output</summary>
```
Found person at [423.33, 55.52, 644.28, 504.13] with score 0.988
- nose: (527.30, 117.12) with score 0.733
- left_eye: (541.79, 109.26) with score 0.554
- right_eye: (515.04, 107.59) with score 0.475
- left_shoulder: (563.30, 171.75) with score 1.000
- right_shoulder: (464.21, 159.75) with score 1.000
- left_elbow: (575.71, 238.04) with score 0.998
- right_elbow: (436.06, 218.10) with score 0.999
- left_wrist: (605.86, 303.35) with score 1.000
- right_wrist: (497.47, 220.82) with score 1.000
- left_hip: (540.97, 307.31) with score 1.000
- right_hip: (475.85, 318.78) with score 1.000
- left_knee: (578.63, 368.63) with score 1.000
- right_knee: (501.05, 442.49) with score 1.000
- left_ankle: (572.11, 464.96) with score 0.991
- right_ankle: (535.75, 441.52) with score 0.981
Found person at [89.97, 3.96, 517.81, 507.28] with score 0.966
- left_shoulder: (242.65, 111.06) with score 0.999
- right_shoulder: (228.79, 112.54) with score 0.999
- left_elbow: (321.84, 169.07) with score 0.999
- right_elbow: (225.76, 218.20) with score 1.000
- left_wrist: (351.73, 220.74) with score 0.999
- right_wrist: (160.19, 228.03) with score 1.000
- left_hip: (342.34, 246.81) with score 1.000
- right_hip: (360.05, 259.35) with score 0.999
- left_knee: (299.56, 377.97) with score 0.998
- right_knee: (313.81, 378.83) with score 0.976
- left_ankle: (443.84, 312.35) with score 0.983
- right_ankle: (424.74, 433.61) with score 0.823
Found person at [-0.53, 51.78, 153.65, 371.17] with score 0.769
- nose: (75.52, 85.67) with score 0.363
- left_shoulder: (121.54, 113.17) with score 1.000
- right_shoulder: (49.77, 117.60) with score 1.000
- left_elbow: (132.90, 147.02) with score 0.932
- right_elbow: (30.31, 156.42) with score 0.992
- left_wrist: (154.43, 162.08) with score 0.871
- right_wrist: (17.20, 196.43) with score 0.943
- left_hip: (105.61, 204.27) with score 0.999
- right_hip: (61.99, 203.66) with score 0.999
- left_knee: (114.70, 270.91) with score 1.000
- right_knee: (63.75, 275.33) with score 1.000
- left_ankle: (125.53, 342.00) with score 0.998
- right_ankle: (63.16, 344.07) with score 0.997
Found person at [519.40, 34.94, 650.11, 312.07] with score 0.488
- nose: (554.82, 76.58) with score 0.920
- left_eye: (563.12, 69.41) with score 0.666
- right_eye: (544.82, 70.01) with score 0.595
- left_shoulder: (596.60, 105.61) with score 0.999
- right_shoulder: (523.29, 107.31) with score 0.969
- left_elbow: (625.14, 151.30) with score 0.999
- right_elbow: (515.96, 147.59) with score 0.322
- left_wrist: (630.90, 196.91) with score 0.998
- right_wrist: (520.75, 181.83) with score 0.415
- left_hip: (583.24, 200.84) with score 0.998
- right_hip: (533.69, 200.01) with score 0.978
- left_knee: (583.79, 265.14) with score 0.934
- right_knee: (538.27, 262.98) with score 0.669
- left_ankle: (584.90, 309.76) with score 0.489
```
</details> |