|
|
|
@ -13,15 +13,14 @@ enum WsDataType { |
|
|
|
// 鼠标操作类型
|
|
|
|
MOUSE_MOVE = "mouse_move", |
|
|
|
MOUSE_CLICK = "mouse_click", |
|
|
|
MOUSE_SCROLL_UP = "mouse_scroll_up", |
|
|
|
MOUSE_SCROLL_DOWN = "mouse_scroll_down", |
|
|
|
|
|
|
|
// 键盘操作类型
|
|
|
|
SEND_KEYS = "send_keys", |
|
|
|
|
|
|
|
// 语音操作类型
|
|
|
|
VOICE_RECORD = "voice_record", |
|
|
|
VOICE_STOP = "voice_stop", |
|
|
|
// 窗口操作类型
|
|
|
|
MOUSE_SCROLL_UP = "mouse_scroll_up", |
|
|
|
MOUSE_SCROLL_DOWN = "mouse_scroll_down", |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
interface WsData { |
|
|
|
@ -81,7 +80,7 @@ export class TriggerAction { |
|
|
|
const message: WsData = { |
|
|
|
type: data.type, |
|
|
|
msg: data.msg || "", |
|
|
|
title: data.title || "Lazyeat", |
|
|
|
title: data.title || "WaveControl", |
|
|
|
duration: data.duration || 1, |
|
|
|
data: data.data || {}, |
|
|
|
}; |
|
|
|
@ -101,17 +100,6 @@ export class TriggerAction { |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
scrollUp() { |
|
|
|
this.send({ |
|
|
|
type: WsDataType.MOUSE_SCROLL_UP, |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
scrollDown() { |
|
|
|
this.send({ |
|
|
|
type: WsDataType.MOUSE_SCROLL_DOWN, |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
sendKeys(key_str: string) { |
|
|
|
this.send({ |
|
|
|
@ -120,15 +108,16 @@ export class TriggerAction { |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
voiceRecord() { |
|
|
|
|
|
|
|
scrollUp() { |
|
|
|
this.send({ |
|
|
|
type: WsDataType.VOICE_RECORD, |
|
|
|
type: WsDataType.MOUSE_SCROLL_UP, |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
voiceStop() { |
|
|
|
scrollDown() { |
|
|
|
this.send({ |
|
|
|
type: WsDataType.VOICE_STOP, |
|
|
|
type: WsDataType.MOUSE_SCROLL_DOWN, |
|
|
|
}); |
|
|
|
} |
|
|
|
} |
|
|
|
@ -146,30 +135,28 @@ export class GestureHandler { |
|
|
|
private previousGestureCount: number = 0; |
|
|
|
private minGestureCount: number = 5; |
|
|
|
|
|
|
|
// 鼠标移动参数
|
|
|
|
// 屏幕参数，用于将摄像头图像坐标映射到实际屏幕坐标
|
|
|
|
private screen_width: number = window.screen.width; |
|
|
|
private screen_height: number = window.screen.height; |
|
|
|
private smoothening = 7; // 平滑系数
|
|
|
|
private prev_loc_x: number = 0; |
|
|
|
private prev_loc_y: number = 0; |
|
|
|
private prev_three_fingers_y: number = 0; // 添加三根手指上一次的 Y 坐标
|
|
|
|
private prev_scroll2_y: number = 0; |
|
|
|
|
|
|
|
// 时间间隔参数
|
|
|
|
private lastClickTime: number = 0; |
|
|
|
private lastScrollTime: number = 0; |
|
|
|
private lastFullScreenTime: number = 0; |
|
|
|
private lastDeleteTime: number = 0; |
|
|
|
|
|
|
|
// 时间间隔常量（毫秒）
|
|
|
|
private readonly CLICK_INTERVAL = 500; // 点击间隔
|
|
|
|
private smoothening = 7; // 平滑系数，用于减少抖动
|
|
|
|
private prev_loc_x: number = 0; // 上一帧鼠标X坐标
|
|
|
|
private prev_loc_y: number = 0; // 上一帧鼠标Y坐标
|
|
|
|
|
|
|
|
private lastClickTime: number = 0; // 上次点击时间戳（用于点击节流）
|
|
|
|
private readonly CLICK_INTERVAL = 500; // 点击最小间隔（ms）
|
|
|
|
|
|
|
|
private lastkeyTime: number = 0; |
|
|
|
private readonly sendkeyINTERVAL = 1500; // 全屏切换间隔
|
|
|
|
|
|
|
|
private app_store: any; // 存储应用状态，比如视频宽高、边界设置等
|
|
|
|
|
|
|
|
private prev_scroll_y: number = 0; |
|
|
|
private lastScrollTime: number = 0; |
|
|
|
private readonly SCROLL_INTERVAL = 100; // 滚动间隔
|
|
|
|
private readonly FULL_SCREEN_INTERVAL = 1500; // 全屏切换间隔
|
|
|
|
|
|
|
|
// 语音识别参数
|
|
|
|
private voice_recording: boolean = false; |
|
|
|
private lastDeleteTime: number = 0; |
|
|
|
|
|
|
|
private app_store: any; |
|
|
|
constructor() { |
|
|
|
this.triggerAction = new TriggerAction(); |
|
|
|
this.app_store = use_app_store(); |
|
|
|
@ -223,16 +210,17 @@ export class GestureHandler { |
|
|
|
this.screen_height |
|
|
|
); |
|
|
|
|
|
|
|
// 应用平滑处理
|
|
|
|
// 加入平滑处理，缓解抖动
|
|
|
|
screenX = |
|
|
|
this.prev_loc_x + (screenX - this.prev_loc_x) / this.smoothening; |
|
|
|
screenY = |
|
|
|
this.prev_loc_y + (screenY - this.prev_loc_y) / this.smoothening; // 消除抖动
|
|
|
|
|
|
|
|
// 更新上一帧坐标
|
|
|
|
this.prev_loc_x = screenX; |
|
|
|
this.prev_loc_y = screenY; |
|
|
|
|
|
|
|
// 移动鼠标
|
|
|
|
// 更新状态，触发鼠标移动指令
|
|
|
|
this.app_store.sub_windows.x = screenX + 10; |
|
|
|
this.app_store.sub_windows.y = screenY; |
|
|
|
this.triggerAction.moveMouse(screenX, screenY); |
|
|
|
@ -242,68 +230,44 @@ export class GestureHandler { |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 处理食指和中指同时竖起手势 - 鼠标左键点击 |
|
|
|
* 食指 + 大拇指上举 → 鼠标左键点击 |
|
|
|
*/ |
|
|
|
private handleMouseClick() { |
|
|
|
const now = Date.now(); |
|
|
|
if (now - this.lastClickTime < this.CLICK_INTERVAL) { |
|
|
|
return; |
|
|
|
return; // 距离上次点击时间太短，忽略（节流）
|
|
|
|
} |
|
|
|
this.lastClickTime = now; |
|
|
|
|
|
|
|
// 发送鼠标点击指令
|
|
|
|
this.triggerAction.clickMouse(); |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 处理三根手指同时竖起手势 - 滚动屏幕 |
|
|
|
* 处理四根手指同时竖起手势 - 发送快捷键 |
|
|
|
*/ |
|
|
|
private handleScroll(hand: HandInfo) { |
|
|
|
const indexTip = this.getFingerTip(hand, 1); |
|
|
|
const middleTip = this.getFingerTip(hand, 2); |
|
|
|
const ringTip = this.getFingerTip(hand, 3); |
|
|
|
if (!indexTip || !middleTip || !ringTip) { |
|
|
|
this.prev_three_fingers_y = 0; |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
const now = Date.now(); |
|
|
|
if (now - this.lastScrollTime < this.SCROLL_INTERVAL) { |
|
|
|
return; |
|
|
|
} |
|
|
|
this.lastScrollTime = now; |
|
|
|
|
|
|
|
// 计算三根手指的平均 Y 坐标
|
|
|
|
const currentY = (indexTip.y + middleTip.y + ringTip.y) / 3; |
|
|
|
|
|
|
|
// 如果是第一次检测到手势，记录当前 Y 坐标
|
|
|
|
if (this.prev_three_fingers_y === 0) { |
|
|
|
this.prev_three_fingers_y = currentY; |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 计算 Y 坐标的变化
|
|
|
|
const deltaY = currentY - this.prev_three_fingers_y; |
|
|
|
|
|
|
|
// 如果变化超过阈值，则触发滚动
|
|
|
|
if (Math.abs(deltaY) > 0.008) { |
|
|
|
if (deltaY < 0) { |
|
|
|
// 手指向上移动，向上滚动
|
|
|
|
this.triggerAction.scrollUp(); |
|
|
|
} else { |
|
|
|
// 手指向下移动，向下滚动
|
|
|
|
this.triggerAction.scrollDown(); |
|
|
|
private handleFourFingers() { |
|
|
|
try { |
|
|
|
const key_str = this.app_store.config.four_fingers_up_send || "f"; |
|
|
|
const now = Date.now(); |
|
|
|
if (now - this.lastkeyTime < this.sendkeyINTERVAL) { |
|
|
|
return; |
|
|
|
} |
|
|
|
// 更新上一次的 Y 坐标
|
|
|
|
this.prev_three_fingers_y = currentY; |
|
|
|
this.lastkeyTime = now; |
|
|
|
|
|
|
|
this.triggerAction.sendKeys(key_str); |
|
|
|
} catch (error) { |
|
|
|
console.error("处理四指手势失败:", error); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// 拇指和食指捏合，滚动屏幕
|
|
|
|
private handleScroll2(hand: HandInfo) { |
|
|
|
private handlescroll(hand: HandInfo) { |
|
|
|
const indexTip = this.getFingerTip(hand, 1); |
|
|
|
const thumbTip = this.getFingerTip(hand, 0); |
|
|
|
if (!indexTip || !thumbTip) { |
|
|
|
this.prev_scroll2_y = 0; |
|
|
|
this.prev_scroll_y = 0; |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
@ -325,18 +289,18 @@ export class GestureHandler { |
|
|
|
distance > |
|
|
|
this.app_store.config.scroll_gesture_2_thumb_and_index_threshold |
|
|
|
) { |
|
|
|
this.prev_scroll2_y = 0; |
|
|
|
this.prev_scroll_y = 0; |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 如果是第一次检测到捏合，记录当前 Y 坐标
|
|
|
|
if (this.prev_scroll2_y === 0) { |
|
|
|
this.prev_scroll2_y = indexTip.y; |
|
|
|
if (this.prev_scroll_y === 0) { |
|
|
|
this.prev_scroll_y = indexTip.y; |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 计算 Y
|
|
|
|
const deltaY = indexTip.y - this.prev_scroll2_y; |
|
|
|
const deltaY = indexTip.y - this.prev_scroll_y; |
|
|
|
|
|
|
|
// 如果变化超过阈值，则触发滚动
|
|
|
|
if (Math.abs(deltaY) > 0.008) { |
|
|
|
@ -348,52 +312,11 @@ export class GestureHandler { |
|
|
|
this.triggerAction.scrollDown(); |
|
|
|
} |
|
|
|
// 更新上一次的 Y 坐标
|
|
|
|
this.prev_scroll2_y = indexTip.y; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 处理四根手指同时竖起手势 - 发送快捷键 |
|
|
|
*/ |
|
|
|
private handleFourFingers() { |
|
|
|
try { |
|
|
|
const key_str = this.app_store.config.four_fingers_up_send || "f"; |
|
|
|
const now = Date.now(); |
|
|
|
if (now - this.lastFullScreenTime < this.FULL_SCREEN_INTERVAL) { |
|
|
|
return; |
|
|
|
} |
|
|
|
this.lastFullScreenTime = now; |
|
|
|
|
|
|
|
this.triggerAction.sendKeys(key_str); |
|
|
|
} catch (error) { |
|
|
|
console.error("处理四指手势失败:", error); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 处理拇指和小指同时竖起手势 - 开始语音识别 |
|
|
|
*/ |
|
|
|
async handleVoiceStart() { |
|
|
|
if (this.voice_recording) { |
|
|
|
return; |
|
|
|
} |
|
|
|
await this.app_store.sub_window_info("开始语音识别"); |
|
|
|
this.voice_recording = true; |
|
|
|
this.triggerAction.voiceRecord(); |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 处理拳头手势 - 停止语音识别 |
|
|
|
*/ |
|
|
|
async handleVoiceStop() { |
|
|
|
if (!this.voice_recording) { |
|
|
|
return; |
|
|
|
this.prev_scroll_y = indexTip.y; |
|
|
|
} |
|
|
|
await this.app_store.sub_window_success("停止语音识别"); |
|
|
|
this.voice_recording = false; |
|
|
|
this.triggerAction.voiceStop(); |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
* 处理删除手势 |
|
|
|
*/ |
|
|
|
@ -433,12 +356,13 @@ export class GestureHandler { |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 获取手指尖点 |
|
|
|
* 获取指定手指的指尖 landmark |
|
|
|
* fingerIndex: 0=拇指, 1=食指, 2=中指, ... |
|
|
|
*/ |
|
|
|
private getFingerTip(hand: HandInfo, fingerIndex: number) { |
|
|
|
if (!hand) return null; |
|
|
|
|
|
|
|
const tipIndices = [4, 8, 12, 16, 20]; |
|
|
|
const tipIndices = [4, 8, 12, 16, 20]; // 手指指尖 landmark 的索引
|
|
|
|
return hand.landmarks[tipIndices[fingerIndex]]; |
|
|
|
} |
|
|
|
|
|
|
|
@ -467,12 +391,6 @@ export class GestureHandler { |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 只要切换手势就停止语音识别
|
|
|
|
if (gesture !== HandGesture.VOICE_GESTURE_START && this.voice_recording) { |
|
|
|
this.handleVoiceStop(); |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 鼠标移动手势直接执行，不需要连续确认
|
|
|
|
if (gesture === HandGesture.ONLY_INDEX_UP) { |
|
|
|
this.handleIndexFingerUp(hand); |
|
|
|
@ -486,17 +404,11 @@ export class GestureHandler { |
|
|
|
this.handleMouseClick(); |
|
|
|
break; |
|
|
|
case HandGesture.SCROLL_GESTURE_2: |
|
|
|
this.handleScroll2(hand); |
|
|
|
this.handlescroll(hand); |
|
|
|
break; |
|
|
|
// case HandGesture.THREE_FINGERS_UP:
|
|
|
|
// this.handleScroll(hand);
|
|
|
|
// break;
|
|
|
|
case HandGesture.FOUR_FINGERS_UP: |
|
|
|
this.handleFourFingers(); |
|
|
|
break; |
|
|
|
case HandGesture.VOICE_GESTURE_START: |
|
|
|
this.handleVoiceStart(); |
|
|
|
break; |
|
|
|
case HandGesture.DELETE_GESTURE: |
|
|
|
this.handleDelete(); |
|
|
|
break; |
|
|
|
|
Backpack
2 months ago