Webカメラだけで物理マウスを捨てる - NonMouseをM2 Mac + Python 3.14で再起動して自分用にチューニングした話

mediapipe==0.8.10
numpy==1.19.3
opencv-python==4.5.1.48
pynput==1.7.4
pyobjc-core==7.3
pyobjc-framework-Cocoa==7.3
# ...

# tested on python==3.13 / 3.14 (Apple Silicon / Intel)
# numpy / protobuf の下限は mediapipe の依存解決に任せる
mediapipe>=0.10.18
numpy
opencv-python>=4.10
opencv-contrib-python>=4.10
pynput>=1.7.7
protobuf
pyinstaller>=6.10
pyobjc-core>=10.3; sys_platform == 'darwin'
pyobjc-framework-Cocoa>=10.3; sys_platform == 'darwin'
pyobjc-framework-Quartz>=10.3; sys_platform == 'darwin'
pyobjc-framework-ApplicationServices>=10.3; sys_platform == 'darwin'

brew install tcl-tk
 
# M2 Mac (Apple Silicon) での例
export PATH="$(brew --prefix tcl-tk)/bin:$PATH"
export LDFLAGS="-L$(brew --prefix tcl-tk)/lib"
export CPPFLAGS="-I$(brew --prefix tcl-tk)/include"
export PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig"
export PYTHON_CONFIGURE_OPTS="--with-tcltk-includes='-I$(brew --prefix tcl-tk)/include' \
  --with-tcltk-libs='-L$(brew --prefix tcl-tk)/lib -ltcl9.0 -ltk9.0'"
 
pyenv install 3.14.0

import mediapipe as mp
from mediapipe.tasks import python as mp_python
from mediapipe.tasks.python import vision as mp_vision
from mediapipe.tasks.python.vision import drawing_utils as mp_drawing
from mediapipe.tasks.python.vision import HandLandmarksConnections
 
# ...
 
hand_options = mp_vision.HandLandmarkerOptions(
    base_options=mp_python.BaseOptions(model_asset_path=MODEL_PATH),
    running_mode=mp_vision.RunningMode.VIDEO,
    num_hands=1,
    min_hand_detection_confidence=HAND_CONFIDENCE,
    min_hand_presence_confidence=HAND_CONFIDENCE,
    min_tracking_confidence=HAND_CONFIDENCE,
)
detector = mp_vision.HandLandmarker.create_from_options(hand_options)

ts_ms = 0
while cap.isOpened():
    # ... (キャプチャ&前処理)
    mp_image = mp.Image(
        image_format=mp.ImageFormat.SRGB,
        data=np.ascontiguousarray(image),
    )
    ts_ms += 1
    results = detector.detect_for_video(mp_image, ts_ms)
    for hand_landmarks in (results.hand_landmarks or []):
        ...

MODEL_URL = (
    "https://storage.googleapis.com/mediapipe-models/"
    "hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"
)
MODEL_PATH = os.path.join(
    os.path.expanduser("~/.cache/nonmouse"), "hand_landmarker.task"
)
 
 
def _ensure_model():
    if not os.path.exists(MODEL_PATH):
        os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
        print(f"Downloading hand landmarker model to {MODEL_PATH} ...")
        urllib.request.urlretrieve(MODEL_URL, MODEL_PATH)

def _ask_int(prompt, default, lo, hi):
    while True:
        raw = input(f"{prompt} [{default}]: ").strip()
        if raw == "":
            return default
        try:
            v = int(raw)
        except ValueError:
            print(f"  -> 整数を入力してください ({lo}-{hi})")
            continue
        if lo <= v <= hi:
            return v
        print(f"  -> {lo}-{hi} の範囲で入力してください")
 
 
def tk_arg():
    print("================ NonMouse Setup ================")
    print("  Enter キーでデフォルト値を採用します")
    print("-------------------------------------------------")
 
    print("[Camera]")
    cap_device = _ask_int("  Device number (0-9)", default=1, lo=0, hi=9)
 
    print("\n[How to place]")
    print("  0 = Normal  (正面から自分を撮影)")
    print("  1 = Above   (手の真上から撮影)")
    print("  2 = Behind  (背後からディスプレイ方向を撮影)")
    mode = _ask_int("  Placement (0-2)", default=1, lo=0, hi=2)
 
    print("\n[Sensitivity]")
    kando_raw = _ask_int("  Sensitivity", default=70, lo=1, hi=100)
    kando = kando_raw / 10
 
    print("\n[Preview]")
    show_preview = _ask_yesno("  Show preview?", default=False)
 
    screenRes = _get_screen_size()
    return cap_device, mode, kando, screenRes, show_preview

def _get_screen_size():
    p = platform.system()
    if p == "Darwin":
        try:
            from Quartz import CGDisplayBounds, CGMainDisplayID
            bounds = CGDisplayBounds(CGMainDisplayID())
            return int(bounds.size.width), int(bounds.size.height)
        except Exception:
            pass
    elif p == "Windows":
        try:
            import ctypes
            user32 = ctypes.windll.user32
            user32.SetProcessDPIAware()
            return user32.GetSystemMetrics(0), user32.GetSystemMetrics(1)
        except Exception:
            pass
    elif p == "Linux":
        try:
            import subprocess, re
            out = subprocess.check_output(["xrandr"], stderr=subprocess.DEVNULL).decode()
            for line in out.splitlines():
                if " connected" in line:
                    m = re.search(r"(\d+)x(\d+)", line)
                    if m:
                        return int(m.group(1)), int(m.group(2))
        except Exception:
            pass
    return 1920, 1080  # フォールバック

from pynput import keyboard as pkeyboard
 
_cmd_pressed = {"v": False}
_alt_pressed = {"v": False}
_click_events = {"left": False, "right": False}
 
 
def _is_cmd(key):
    if platform.system() == "Darwin":
        return key in (pkeyboard.Key.cmd, pkeyboard.Key.cmd_l, pkeyboard.Key.cmd_r)
    else:
        # Linux/Windows には Command がないので Alt で代替
        return key in (pkeyboard.Key.alt, pkeyboard.Key.alt_l, pkeyboard.Key.alt_r)
 
 
def _is_alt(key):
    if platform.system() == "Darwin":
        return key in (pkeyboard.Key.alt, pkeyboard.Key.alt_l, pkeyboard.Key.alt_r)
    else:
        return key in (pkeyboard.Key.ctrl, pkeyboard.Key.ctrl_l, pkeyboard.Key.ctrl_r)
 
 
def _on_press(key):
    if _is_cmd(key):
        _cmd_pressed["v"] = True
    if _is_alt(key):
        _alt_pressed["v"] = True
    # 左 Shift でも左クリック発火 (cmd 押下中のみ)
    if key == pkeyboard.Key.shift_l and _cmd_pressed["v"]:
        _click_events["left"] = True
    ch = getattr(key, "char", None)
    if ch == "k" and _cmd_pressed["v"]:
        _click_events["right"] = True
 
 
def _on_release(key):
    if _is_cmd(key):
        _cmd_pressed["v"] = False
    if _is_alt(key):
        _alt_pressed["v"] = False
 
 
key_listener = pkeyboard.Listener(on_press=_on_press, on_release=_on_release)
key_listener.daemon = True
key_listener.start()

cursor_mode = _cmd_pressed["v"]
scroll_mode = _alt_pressed["v"] and not cursor_mode
if not cursor_mode:
    _click_events["left"] = False
    _click_events["right"] = False
    i = 0            # カーソル再アンカー待ち
if not scroll_mode:
    prev_middle_y = None

if cursor_mode:
    if i == 0:
        for _lst in (LiTx, LiTy, list0x, list0y, list1x, list1y):
            _lst.clear()
        preX = hand_landmarks[9].x
        preY = hand_landmarks[9].y
        i += 1
 
    nowX = calculate_moving_average(hand_landmarks[9].x, ran, LiTx)
    nowY = calculate_moving_average(hand_landmarks[9].y, ran, LiTy)
    raw_dx = (nowX - preX) * image_width
    raw_dy = (nowY - preY) * image_height
    preX = nowX
    preY = nowY

# カーソル移動の加速カーブ設定
DEADZONE_PX = 0.2        # これ以下の指移動は無視 (ジッター抑制)
ACCEL_POWER = 1.4        # 非線形加速指数 (>1 で大きく振るほど指数的に速くなる)
 
# ...
speed = float(np.hypot(raw_dx, raw_dy))
if speed <= DEADZONE_PX:
    dx = 0.0
    dy = 0.0
else:
    effective = speed - DEADZONE_PX
    amplified = base_gain * (effective ** ACCEL_POWER)
    scale = amplified / speed
    dx = raw_dx * scale
    dy = raw_dy * scale

# 実測: 右/左 out_x=55.5, 上/下 out_y=61.2 → x を 10% ブースト
AXIS_GAIN_X = 1.10
AXIS_GAIN_Y = 1.00
 
# ...
dx *= AXIS_GAIN_X
dy *= AXIS_GAIN_Y

# 方向別ゲイン: 実測 上 out_y=69.78, 下 out_y=52.55 → 下方向を 1.33 倍
GAIN_Y_UP = 1.00
GAIN_Y_DOWN = 1.33
GAIN_X_LEFT = 1.00
GAIN_X_RIGHT = 1.00
 
# ...
if dx >= 0:
    dx *= GAIN_X_RIGHT
else:
    dx *= GAIN_X_LEFT
if dy >= 0:
    dy *= GAIN_Y_DOWN   # 画像座標系は下が正
else:
    dy *= GAIN_Y_UP

pos_before = mouse.position
mouse.move(dx, dy)
pos_after = mouse.position
actual_dx = pos_after[0] - pos_before[0]
actual_dy = pos_after[1] - pos_before[1]
 
# dx が有意に出ているのに画面は動いていない → その向きは画面端
# preX をスナップし、次フレーム以降その方向への累積を止める
if abs(dx) >= 1.0:
    if (dx > 0 and actual_dx <= 0) or (dx < 0 and actual_dx >= 0):
        preX = hand_landmarks[9].x
if abs(dy) >= 1.0:
    if (dy > 0 and actual_dy <= 0) or (dy < 0 and actual_dy >= 0):
        preY = hand_landmarks[9].y

INVERT_CURSOR_X = True
INVERT_CURSOR_Y = True
 
# ...
if INVERT_CURSOR_X:
    raw_dx = -raw_dx
if INVERT_CURSOR_Y:
    raw_dy = -raw_dy

# ハンド検出/追跡の信頼度 (大きいほど誤検出が減る)
HAND_CONFIDENCE = 0.9
 
# スムージング窓フレーム数 (大きいほど滑らかだが遅延が増える)
SMOOTHING_FRAMES = 6
 
# これ以下の指移動は無視 (加速前のデッドゾーン)
DEADZONE_PX = 0.2
 
# 最終的な dx/dy がこれ未満ならゼロに丸める
OUTPUT_MIN_PX = 0.0

hud_lines = [
    (f"MODE  : {mode_str}", mode_color),
    (f"ACTIVE: {active_str}", (0, 255, 255)),
    ("-- CURSOR ({}) --".format(CURSOR_HOTKEY_NAME), (255, 255, 0)),
    (f"raw   : ({dbg['raw'][0]:+6.2f},{dbg['raw'][1]:+6.2f}) px", (255, 255, 0)),
    (f"dx/dy : ({dbg['dxdy'][0]:+6.2f},{dbg['dxdy'][1]:+6.2f}) px", (255, 255, 0)),
    (f"speed : {dbg['speed']:6.2f}  gain={base_gain:.2f}", (255, 255, 0)),
    (f"avgRaw: ({dbg['avg_raw'][0]:+6.2f},{dbg['avg_raw'][1]:+6.2f}) |abs|=({dbg['avg_abs_raw'][0]:5.2f},{dbg['avg_abs_raw'][1]:5.2f})", (200, 200, 255)),
    (f"avgOut: ({dbg['avg_dxdy'][0]:+6.2f},{dbg['avg_dxdy'][1]:+6.2f}) |abs|=({dbg['avg_abs_dxdy'][0]:5.2f},{dbg['avg_abs_dxdy'][1]:5.2f})", (200, 200, 255)),
    (f"axis  : x={AXIS_GAIN_X:.2f} y={AXIS_GAIN_Y:.2f}", (200, 200, 255)),
    (f"dir   : L={GAIN_X_LEFT:.2f} R={GAIN_X_RIGHT:.2f} U={GAIN_Y_UP:.2f} D={GAIN_Y_DOWN:.2f}", (200, 200, 255)),
    (f"cursor: {'ACTIVE' if act['cursor'] else 'idle'}", (0, 255, 0) if act['cursor'] else (180, 180, 180)),
    # ... CLICK / SCROLL セクション
]

ACTIVE_HOLD_S = 0.4
active_until = {"cursor": 0.0, "lclick": 0.0, "rclick": 0.0, "scroll": 0.0}
 
# 発火時にタイムスタンプを書き込む
if _click_events["left"]:
    _click_events["left"] = False
    mouse.click(Button.left)
    active_until["lclick"] = time.perf_counter() + ACTIVE_HOLD_S
 
# HUD 描画時に判定
_now = time.perf_counter()
act = {k: 1 if _now < v else 0 for k, v in active_until.items()}

if show_preview:
    _draw_circle = draw_circle
else:
    def _draw_circle(*_a, **_kw):
        pass