diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 82f205fea..275976705 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -20,7 +20,7 @@ jobs: node-version: ${{ matrix.node-version }} - uses: pnpm/action-setup@v4 with: - version: 8 + version: 9 - name: install & build run: pnpm build env: @@ -30,7 +30,7 @@ jobs: run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch - name: Deploy 🚀 - uses: JamesIves/github-pages-deploy-action@v4 + uses: JamesIves/github-pages-deploy-action@v4.6.4 with: folder: build # The folder the action should deploy. target-folder: ${{ steps.extract_branch.outputs.branch }} diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 5e566d404..2480dfb54 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -1,7 +1,6 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent } from "$common"; import { legacyFullSupport, info } from "./legacy"; - -import * as handpose from '@tensorflow-models/handpose'; +import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; const { legacyExtension, legacyDefinition } = legacyFullSupport.for(); // TODO: Add extension's health check (peripheral) @@ -82,12 +81,23 @@ export default class PoseHand extends Extension { * @param env */ init(env: Environment) { - + this.loadMediaPipeModel(); if (this.runtime.ioDevices) { this._loop(); } } + /** + * Converts the coordinates from the MediaPipe hand estimate to Scratch coordinates + * @param x + * @param y + * @param z + * @returns enum + */ + mediapipeCoordsToScratch(x, y, z) { + return this.tfCoordsToScratch({ x: this.DIMENSIONS[0] * x, y: this.DIMENSIONS[1] * y, z }); + } + /** * Converts the coordinates from the hand pose estimate to Scratch coordinates * @param x @@ -113,8 +123,7 @@ export default class PoseHand extends Extension { * @returns {boolean} true if connected, false if not connected */ isConnected() { - console.log('connected'); - return !!this.handPoseState && this.handPoseState.length > 0; + return !!this.handPoseState && this.handPoseState.landmarks.length > 0; } /** @@ -125,38 +134,34 @@ export default class PoseHand extends Extension { async _loop() { while (true) { const frame = this.runtime.ioDevices.video.getFrame({ - format: 'image-data', + format: 'canvas', dimensions: this.DIMENSIONS }); const time = +new Date(); - if (frame) { - this.handPoseState = await this.estimateHandPoseOnImage(frame); + if (this.handModel && frame) { + this.handPoseState = this.handModel.detect(frame); } const estimateThrottleTimeout = (+new Date() - time) / 4; await new Promise(r => setTimeout(r, estimateThrottleTimeout)); } } - /** - * Estimates where the hand is on the video frame. - * @param imageElement - * @returns {Promise} - */ - async estimateHandPoseOnImage(imageElement) { - const handModel = await this.getLoadedHandModel(); - return await handModel.estimateHands(imageElement, { - flipHorizontal: false - }); - } - /** - * Gets the hand model from handpose - * @returns hand model - */ - async getLoadedHandModel() { - this.handModel ??= await handpose.load(); - return this.handModel; + + async loadMediaPipeModel() { + const vision = await FilesetResolver.forVisionTasks( + // path/to/wasm/root + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + this.handModel = await HandLandmarker.createFromOptions( + vision, + { + baseOptions: { + modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task" + }, + numHands: 2 + }); } /** @@ -196,12 +201,44 @@ export default class PoseHand extends Extension { const handlerFingerOptions: Array = this.fingerOptions.map(finger => finger.value); + const handOptions = { + "thumb": { + 3: 4, + 1: 2, + 0: 1, + 2: 3 + }, + "indexFinger": { + 3: 8, + 1: 6, + 0: 5, + 2: 7 + }, + "middleFinger": { + 3: 12, + 1: 10, + 0: 9, + 2: 11 + }, + "ringFinger": { + 3: 16, + 1: 14, + 0: 13, + 2: 15 + }, + "pinky": { + 3: 20, + 1: 18, + 0: 17, + 2: 19 + }, + } + const goToHandPart = legacyDefinition.goToHandPart({ operation: (handPart: string, fingerPart: number, util) => { if (this.isConnected()) { - console.log('connected 2'); - const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; - const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z }); + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); (util.target as any).setXY(scratchX, scratchY, false); } }, diff --git a/extensions/src/poseHand/package.json b/extensions/src/poseHand/package.json index 32b84239f..629670275 100644 --- a/extensions/src/poseHand/package.json +++ b/extensions/src/poseHand/package.json @@ -11,6 +11,7 @@ "author": "", "license": "ISC", "dependencies": { + "@mediapipe/tasks-vision": "^0.1.0-alpha-12", "@tensorflow-models/handpose": "^0.0.3" } } \ No newline at end of file diff --git a/extensions/src/poseHand/pnpm-lock.yaml b/extensions/src/poseHand/pnpm-lock.yaml index d1084d72f..247ecd233 100644 --- a/extensions/src/poseHand/pnpm-lock.yaml +++ b/extensions/src/poseHand/pnpm-lock.yaml @@ -8,12 +8,18 @@ importers: .: dependencies: + '@mediapipe/tasks-vision': + specifier: ^0.1.0-alpha-12 + version: 0.1.0-alpha-9 '@tensorflow-models/handpose': specifier: ^0.0.3 version: 0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4) packages: + '@mediapipe/tasks-vision@0.1.0-alpha-9': + resolution: {integrity: sha512-hpgY13d3zwbKEyEEG03m1rpEYM56CGPcMGgHwveZ4+SfDjmMLaev14bvlVpog51UNDE7abJTRL6Q4OtIJsSxXQ==} + '@tensorflow-models/handpose@0.0.3': resolution: {integrity: sha512-U5SBwxeQUXVawACDn+e0r4XJEDEah/J1HlWAqApXcm8DXjCtGKxQm/8BmFsg6ebbtAQ/R1bripohaQ655fv29w==} peerDependencies: @@ -50,6 +56,8 @@ packages: snapshots: + '@mediapipe/tasks-vision@0.1.0-alpha-9': {} + '@tensorflow-models/handpose@0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4)': dependencies: '@tensorflow/tfjs-converter': 1.7.4(@tensorflow/tfjs-core@1.7.4) diff --git a/package.json b/package.json index 44fd3ca97..74eea4912 100644 --- a/package.json +++ b/package.json @@ -20,12 +20,12 @@ "etest": "pnpm test:extensions" }, "devDependencies": { - "@types/node": "^20.12.2", + "@types/node": "^20.16.10", "@types/webgl2": "^0.0.6", - "@types/yargs": "^17.0.32", - "ts-node": "^10.9.1", - "tsconfig-paths": "^4.1.1", - "tslib": "^2.4.1", + "@types/yargs": "^17.0.33", + "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", + "tslib": "^2.7.0", "typescript": "latest", "yargs": "^17.7.2" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40e737ba1..43d79e9a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,26 +9,26 @@ importers: .: devDependencies: '@types/node': - specifier: ^20.12.2 - version: 20.12.12 + specifier: ^20.16.10 + version: 20.16.10 '@types/webgl2': specifier: ^0.0.6 version: 0.0.6 '@types/yargs': - specifier: ^17.0.32 - version: 17.0.32 + specifier: ^17.0.33 + version: 17.0.33 ts-node: - specifier: ^10.9.1 - version: 10.9.2(@types/node@20.12.12)(typescript@5.4.5) + specifier: ^10.9.2 + version: 10.9.2(@types/node@20.16.10)(typescript@5.6.2) tsconfig-paths: - specifier: ^4.1.1 + specifier: ^4.2.0 version: 4.2.0 tslib: - specifier: ^2.4.1 - version: 2.6.2 + specifier: ^2.7.0 + version: 2.7.0 typescript: specifier: latest - version: 5.4.5 + version: 5.6.2 yargs: specifier: ^17.7.2 version: 17.7.2 @@ -43,8 +43,8 @@ packages: resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} - '@jridgewell/sourcemap-codec@1.4.15': - resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + '@jridgewell/sourcemap-codec@1.5.0': + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -61,8 +61,8 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} - '@types/node@20.12.12': - resolution: {integrity: sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==} + '@types/node@20.16.10': + resolution: {integrity: sha512-vQUKgWTjEIRFCvK6CyriPH3MZYiYlNy0fKiEYHWbcoWLEgs4opurGGKlebrTLqdSMIbXImH6XExNiIyNUv3WpA==} '@types/webgl2@0.0.6': resolution: {integrity: sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==} @@ -70,15 +70,15 @@ packages: '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} - '@types/yargs@17.0.32': - resolution: {integrity: sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==} + '@types/yargs@17.0.33': + resolution: {integrity: sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==} - acorn-walk@8.3.2: - resolution: {integrity: sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==} + acorn-walk@8.3.4: + resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} engines: {node: '>=0.4.0'} - acorn@8.11.3: - resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} + acorn@8.12.1: + resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==} engines: {node: '>=0.4.0'} hasBin: true @@ -114,8 +114,8 @@ packages: emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} - escalade@3.1.2: - resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} get-caller-file@2.0.5: @@ -171,16 +171,16 @@ packages: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} - tslib@2.6.2: - resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + tslib@2.7.0: + resolution: {integrity: sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==} - typescript@5.4.5: - resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==} + typescript@5.6.2: + resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==} engines: {node: '>=14.17'} hasBin: true - undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.19.8: + resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -213,12 +213,12 @@ snapshots: '@jridgewell/resolve-uri@3.1.2': {} - '@jridgewell/sourcemap-codec@1.4.15': {} + '@jridgewell/sourcemap-codec@1.5.0': {} '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@tsconfig/node10@1.0.11': {} @@ -228,21 +228,23 @@ snapshots: '@tsconfig/node16@1.0.4': {} - '@types/node@20.12.12': + '@types/node@20.16.10': dependencies: - undici-types: 5.26.5 + undici-types: 6.19.8 '@types/webgl2@0.0.6': {} '@types/yargs-parser@21.0.3': {} - '@types/yargs@17.0.32': + '@types/yargs@17.0.33': dependencies: '@types/yargs-parser': 21.0.3 - acorn-walk@8.3.2: {} + acorn-walk@8.3.4: + dependencies: + acorn: 8.12.1 - acorn@8.11.3: {} + acorn@8.12.1: {} ansi-regex@5.0.1: {} @@ -270,7 +272,7 @@ snapshots: emoji-regex@8.0.0: {} - escalade@3.1.2: {} + escalade@3.2.0: {} get-caller-file@2.0.5: {} @@ -296,21 +298,21 @@ snapshots: strip-bom@3.0.0: {} - ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5): + ts-node@10.9.2(@types/node@20.16.10)(typescript@5.6.2): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.12.12 - acorn: 8.11.3 - acorn-walk: 8.3.2 + '@types/node': 20.16.10 + acorn: 8.12.1 + acorn-walk: 8.3.4 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.4.5 + typescript: 5.6.2 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 @@ -320,11 +322,11 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 - tslib@2.6.2: {} + tslib@2.7.0: {} - typescript@5.4.5: {} + typescript@5.6.2: {} - undici-types@5.26.5: {} + undici-types@6.19.8: {} v8-compile-cache-lib@3.0.1: {} @@ -341,7 +343,7 @@ snapshots: yargs@17.7.2: dependencies: cliui: 8.0.1 - escalade: 3.1.2 + escalade: 3.2.0 get-caller-file: 2.0.5 require-directory: 2.1.1 string-width: 4.2.3