You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
272 lines
7.1 KiB
272 lines
7.1 KiB
import { GestureHandler } from "@/hand_landmark/gesture_handler";
|
|
import {
|
|
FilesetResolver,
|
|
GestureRecognizer,
|
|
GestureRecognizerOptions,
|
|
} from "@mediapipe/tasks-vision";
|
|
|
|
// 手势
|
|
export enum HandGesture {
|
|
// 食指举起，移动鼠标
|
|
ONLY_INDEX_UP = "only_index_up",
|
|
|
|
// 食指和拇指举起，移动鼠标
|
|
INDEX_AND_THUMB_UP = "index_and_thumb_up",
|
|
|
|
// ok手势 - 滚动屏幕
|
|
SCROLL_GESTURE = "scroll_gesture",
|
|
|
|
// 四根手指同时竖起 - enter
|
|
FOUR_FINGERS_UP = "four_fingers_up",
|
|
|
|
// 五根手指同时竖起 - 暂停/开始 识别
|
|
STOP_GESTURE = "stop_gesture",
|
|
|
|
// 6手势 - 语音识别
|
|
VOICE_GESTURE_START = "voice_gesture_start",
|
|
VOICE_GESTURE_STOP = "voice_gesture_stop",
|
|
|
|
// 其他手势
|
|
OTHER = "other",
|
|
|
|
}
|
|
|
|
interface HandLandmark {
|
|
x: number;
|
|
y: number;
|
|
z: number;
|
|
}
|
|
|
|
export interface HandInfo {
|
|
landmarks: HandLandmark[];
|
|
handedness: "Left" | "Right";
|
|
score: number;
|
|
categoryName?: string;
|
|
}
|
|
|
|
interface DetectionResult {
|
|
leftHand?: HandInfo;
|
|
rightHand?: HandInfo;
|
|
// 原始检测结果，以防需要访问其他数据
|
|
rawResult: any;
|
|
}
|
|
|
|
/**
|
|
* 检测器类 - 负责手势识别和手势分类
|
|
* 主要职责:
|
|
* 1. 初始化
|
|
* 2. 检测视频帧中的手部
|
|
* 3. 分析手势类型(手指竖起等)
|
|
* 4. 提供手部关键点查询方法
|
|
*/
|
|
export class Detector {
|
|
private detector: GestureRecognizer | null = null;
|
|
private gestureHandler: GestureHandler | null = null;
|
|
|
|
async initialize(useCanvas = false) {
|
|
const vision = await FilesetResolver.forVisionTasks("/mediapipe/wasm");
|
|
try {
|
|
const params = {
|
|
baseOptions: {
|
|
modelAssetPath: "/mediapipe/gesture_recognizer.task",
|
|
delegate: "GPU",
|
|
},
|
|
runningMode: "VIDEO",
|
|
numHands: 1,
|
|
} as GestureRecognizerOptions;
|
|
if (useCanvas) {
|
|
params.canvas = document.createElement("canvas");
|
|
}
|
|
|
|
this.detector = await GestureRecognizer.createFromOptions(vision, params);
|
|
this.gestureHandler = new GestureHandler();
|
|
} catch (error: any) {
|
|
if (error.toString().includes("kGpuService")) {
|
|
await this.initialize(true);
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
// 检测视频帧中的手部（保持独立）
|
|
async detect(video: HTMLVideoElement): Promise<DetectionResult> {
|
|
if (!this.detector) {
|
|
throw new Error("检测器未初始化");
|
|
}
|
|
|
|
const now = performance.now();
|
|
const result = await this.detector.recognizeForVideo(video, now);
|
|
const detection: DetectionResult = {
|
|
rawResult: result,
|
|
};
|
|
|
|
if (result.landmarks && result.handedness) {
|
|
for (let i = 0; i < result.landmarks.length; i++) {
|
|
const hand: HandInfo = {
|
|
landmarks: result.landmarks[i],
|
|
handedness: result.handedness[i][0].categoryName as "Left" | "Right",
|
|
score: result.handedness[i][0].score,
|
|
};
|
|
|
|
if (result.gestures.length > 0) {
|
|
hand.categoryName = result.gestures[0][0].categoryName;
|
|
}
|
|
|
|
if (hand.handedness === "Left") {
|
|
detection.leftHand = hand;
|
|
} else {
|
|
detection.rightHand = hand;
|
|
}
|
|
}
|
|
}
|
|
|
|
return detection;
|
|
}
|
|
|
|
/**
|
|
* 检测手指是否竖起
|
|
*/
|
|
static _fingersUp(hand: HandInfo): number[] {
|
|
const fingers: number[] = [];
|
|
const tipIds = [4, 8, 12, 16, 20]; // 从大拇指开始，依次为每个手指指尖
|
|
|
|
// 检测大拇指
|
|
if (hand.handedness === "Right") {
|
|
if (hand.landmarks[tipIds[0]].x < hand.landmarks[tipIds[0] - 1].x) {
|
|
fingers.push(0);
|
|
} else {
|
|
fingers.push(1);
|
|
}
|
|
} else {
|
|
if (hand.landmarks[tipIds[0]].x > hand.landmarks[tipIds[0] - 1].x) {
|
|
fingers.push(0);
|
|
} else {
|
|
fingers.push(1);
|
|
}
|
|
}
|
|
|
|
// 检测其他四个手指
|
|
for (let id = 1; id < 5; id++) {
|
|
if (hand.landmarks[tipIds[id]].y < hand.landmarks[tipIds[id] - 2].y) {
|
|
fingers.push(1);
|
|
} else {
|
|
fingers.push(0);
|
|
}
|
|
}
|
|
|
|
return fingers;
|
|
}
|
|
|
|
/**
|
|
* 获取单个手的手势类型
|
|
*/
|
|
public static getSingleHandGesture(hand: HandInfo): HandGesture {
|
|
const fingers = this._fingersUp(hand);
|
|
const fingerState = fingers.join(",");
|
|
|
|
// 定义手势映射表
|
|
const gestureMap = new Map<string, HandGesture>([
|
|
// 食指举起，移动鼠标
|
|
["0,1,0,0,0", HandGesture.ONLY_INDEX_UP],
|
|
|
|
// 鼠标左键点击手势
|
|
["1,1,0,0,0", HandGesture.INDEX_AND_THUMB_UP],
|
|
|
|
// 滚动屏幕手势 ok
|
|
["0,0,1,1,1", HandGesture.SCROLL_GESTURE],
|
|
|
|
// 四根手指同时竖起
|
|
["0,1,1,1,1", HandGesture.FOUR_FINGERS_UP],
|
|
|
|
// 五根手指同时竖起 - 暂停/开始 识别
|
|
["1,1,1,1,1", HandGesture.STOP_GESTURE],
|
|
|
|
// 6手势- 语音识别
|
|
["1,0,0,0,1", HandGesture.VOICE_GESTURE_START],
|
|
|
|
// 结束语音识别
|
|
["0,0,0,0,0", HandGesture.VOICE_GESTURE_STOP],
|
|
|
|
|
|
]);
|
|
|
|
if (gestureMap.has(fingerState)) {
|
|
return gestureMap.get(fingerState) as HandGesture;
|
|
}
|
|
|
|
|
|
// 返回默认值
|
|
return HandGesture.OTHER;
|
|
}
|
|
|
|
/**
|
|
* 处理检测结果并执行相应动作
|
|
*/
|
|
//async process(detection: DetectionResult): Promise<HandGesture> {
|
|
// const hand = detection.rightHand ?? detection.leftHand;
|
|
|
|
// if (!hand) {
|
|
// console.log("没检测到手");
|
|
// return HandGesture.OTHER;
|
|
// }
|
|
|
|
// const gesture = Detector.getSingleHandGesture(hand);
|
|
// console.log("当前手势状态是：", gesture);
|
|
|
|
// if (gesture === HandGesture.ONLY_INDEX_UP) {
|
|
// console.log("识别到食指竖起！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.INDEX_AND_THUMB_UP) {
|
|
// console.log("识别到食指和大拇指竖起！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.THREE_FINGERS_UP) {
|
|
// console.log("识别到ok手势！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.FOUR_FINGERS_UP) {
|
|
// console.log("识别到四指竖起！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.STOP_GESTURE) {
|
|
// console.log("识别到五指竖起！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.VOICE_GESTURE_START) {
|
|
// console.log("识别到6手势！");
|
|
// }
|
|
|
|
// if (gesture === HandGesture.VOICE_GESTURE_STOP) {
|
|
// console.log("识别到拳头手势！");
|
|
// }
|
|
|
|
// return gesture;
|
|
async process(detection: DetectionResult): Promise<void> {
|
|
const rightHandGesture = detection.rightHand
|
|
? Detector.getSingleHandGesture(detection.rightHand)
|
|
: HandGesture.OTHER;
|
|
const leftHandGesture = detection.leftHand
|
|
? Detector.getSingleHandGesture(detection.leftHand)
|
|
: HandGesture.OTHER;
|
|
|
|
// 优先使用右手
|
|
let effectiveGesture = rightHandGesture;
|
|
if (detection.rightHand) {
|
|
effectiveGesture = rightHandGesture;
|
|
} else if (detection.leftHand) {
|
|
effectiveGesture = leftHandGesture;
|
|
}
|
|
|
|
// 将手势处理交给GestureHandler
|
|
if (detection.rightHand) {
|
|
this.gestureHandler?.handleGesture(effectiveGesture, detection.rightHand);
|
|
} else if (detection.leftHand) {
|
|
this.gestureHandler?.handleGesture(effectiveGesture, detection.leftHand);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
}
|