Mediapipe hand

.github/workflows/build-and-deploy.yml

@@ -20,7 +20,7 @@ jobs:
  node-version: ${{ matrix.node-version }}
  - uses: pnpm/action-setup@v4
  with:
- version: 8
+ version: 9
  - name: install & build
  run: pnpm build
  env:
@@ -30,7 +30,7 @@ jobs:
  run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
  id: extract_branch
  - name: Deploy 🚀
- uses: JamesIves/github-pages-deploy-action@v4
+ uses: JamesIves/github-pages-deploy-action@v4.6.4
  with:
  folder: build # The folder the action should deploy.
  target-folder: ${{ steps.extract_branch.outputs.branch }}

extensions/src/poseHand/index.ts

@@ -1,7 +1,6 @@
 import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent } from "$common";
 import { legacyFullSupport, info } from "./legacy";
-
-import * as handpose from '@tensorflow-models/handpose';
+import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
 const { legacyExtension, legacyDefinition } = legacyFullSupport.for<PoseHand>();
 
 // TODO: Add extension's health check (peripheral)
@@ -82,12 +81,23 @@ export default class PoseHand extends Extension<Details, Blocks> {
  * @param env 
  */
  init(env: Environment) {
-
+ this.loadMediaPipeModel();
  if (this.runtime.ioDevices) {
  this._loop();
  }
  }
 
+ /**
+ * Converts the coordinates from the MediaPipe hand estimate to Scratch coordinates
+ * @param x 
+ * @param y
+ * @param z
+ * @returns enum
+ */
+ mediapipeCoordsToScratch(x, y, z) {
+ return this.tfCoordsToScratch({ x: this.DIMENSIONS[0] * x, y: this.DIMENSIONS[1] * y, z });
+ }
+
  /**
  * Converts the coordinates from the hand pose estimate to Scratch coordinates
  * @param x 
@@ -113,8 +123,7 @@ export default class PoseHand extends Extension<Details, Blocks> {
  * @returns {boolean} true if connected, false if not connected
  */
  isConnected() {
- console.log('connected');
- return !!this.handPoseState && this.handPoseState.length > 0;
+ return !!this.handPoseState && this.handPoseState.landmarks.length > 0;
  }
 
  /**
@@ -125,38 +134,34 @@ export default class PoseHand extends Extension<Details, Blocks> {
  async _loop() {
  while (true) {
  const frame = this.runtime.ioDevices.video.getFrame({
- format: 'image-data',
+ format: 'canvas',
  dimensions: this.DIMENSIONS
  });
 
  const time = +new Date();
- if (frame) {
- this.handPoseState = await this.estimateHandPoseOnImage(frame);
+ if (this.handModel && frame) {
+ this.handPoseState = this.handModel.detect(frame);
  }
  const estimateThrottleTimeout = (+new Date() - time) / 4;
  await new Promise(r => setTimeout(r, estimateThrottleTimeout));
  }
  }
 
- /**
- * Estimates where the hand is on the video frame.
- * @param imageElement
- * @returns {Promise<AnnotatedPrediction[]>}
- */
- async estimateHandPoseOnImage(imageElement) {
- const handModel = await this.getLoadedHandModel();
- return await handModel.estimateHands(imageElement, {
- flipHorizontal: false
- });
- }
 
- /**
- * Gets the hand model from handpose
- * @returns hand model
- */
- async getLoadedHandModel() {
- this.handModel ??= await handpose.load();
- return this.handModel;
+
+ async loadMediaPipeModel() {
+ const vision = await FilesetResolver.forVisionTasks(
+ // path/to/wasm/root
+ "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
+ );
+ this.handModel = await HandLandmarker.createFromOptions(
+ vision,
+ {
+ baseOptions: {
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task"
+ },
+ numHands: 2
+ });
  }
 
  /**
@@ -196,12 +201,44 @@ export default class PoseHand extends Extension<Details, Blocks> {
 
  const handlerFingerOptions: Array<string> = this.fingerOptions.map(finger => finger.value);
 
+ const handOptions = {
+ "thumb": {
+ 3: 4,
+ 1: 2,
+ 0: 1,
+ 2: 3
+ },
+ "indexFinger": {
+ 3: 8,
+ 1: 6,
+ 0: 5,
+ 2: 7
+ },
+ "middleFinger": {
+ 3: 12,
+ 1: 10,
+ 0: 9,
+ 2: 11
+ },
+ "ringFinger": {
+ 3: 16,
+ 1: 14,
+ 0: 13,
+ 2: 15
+ },
+ "pinky": {
+ 3: 20,
+ 1: 18,
+ 0: 17,
+ 2: 19
+ },
+ }
+
  const goToHandPart = legacyDefinition.goToHandPart({
  operation: (handPart: string, fingerPart: number, util) => {
  if (this.isConnected()) {
- console.log('connected 2');
- const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart];
- const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z });
+ const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]];
+ const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z);
  (util.target as any).setXY(scratchX, scratchY, false);
  }
  },

extensions/src/poseHand/package.json

@@ -11,6 +11,7 @@
  "author": "",
  "license": "ISC",
  "dependencies": {
+ "@mediapipe/tasks-vision": "^0.1.0-alpha-12",
  "@tensorflow-models/handpose": "^0.0.3"
  }
 }

package.json

@@ -20,12 +20,12 @@
  "etest": "pnpm test:extensions"
  },
  "devDependencies": {
- "@types/node": "^20.12.2",
+ "@types/node": "^20.16.10",
  "@types/webgl2": "^0.0.6",
- "@types/yargs": "^17.0.32",
- "ts-node": "^10.9.1",
- "tsconfig-paths": "^4.1.1",
- "tslib": "^2.4.1",
+ "@types/yargs": "^17.0.33",
+ "ts-node": "^10.9.2",
+ "tsconfig-paths": "^4.2.0",
+ "tslib": "^2.7.0",
  "typescript": "latest",
  "yargs": "^17.7.2"
  },