当前位置： 首页 > article >正文

亚博microros小车-原生ubuntu支持系列：10-画笔

article 2025/3/5 6:27:16

背景知识

基础框架还是用到了MediaPipe Hands，亚博microros小车-原生ubuntu支持系列：4-手部检测-CSDN博客

为了方便对照，还是贴一下手部的图

其中，手指头尖是4，8，12，16，20.

检测到手指头是否漏出。

    #手指头漏出检测
    def fingersUp(self):
        fingers=[]
        # Thumb
        if (self.calc_angle(self.tipIds[0],
                            self.tipIds[0] - 1,
                            self.tipIds[0] - 2) > 150.0) and (
                self.calc_angle(
                    self.tipIds[0] - 1,
                    self.tipIds[0] - 2,
                    self.tipIds[0] - 3) > 150.0): fingers.append(1)
        else:
            fingers.append(0)
        # 4 finger
        for id in range(1, 5):
            if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] - 2][2]:
                fingers.append(1)
            else:
                fingers.append(0)
        return fingers

对于拇指，是点4，3，2和3，2，1点的计算角度。

余弦定理：c2 = a2 + b2 − 2ab cos(C)

先算了点的坐标，计算欧式距离，在根据余弦定理去计算。认为角432>150度和角321>150度是大拇指伸出.其余四指检测:是指尖点与-2点的,比如8跟6,12跟10比，坐标小的则认为伸出

lmList:[[0, 101, 392], [1, 161, 368], [2, 213, 318], [3, 246, 275], [4, 240, 232], [5, 188, 210], [6, 220, 134], [7, 239, 90], [8, 252, 50], [9, 142, 210], [10, 170, 139], [11, 189, 196], [12, 191, 245], [13, 95, 225], [14, 112, 174], [15, 136, 234], [16, 142, 277], [17, 51, 254], [18, 71, 218], [19, 97, 259], [20, 104, 292]]
fingers :[0, 1, 0, 0, 0]
lmList:[[0, 110, 381], [1, 169, 354], [2, 210, 298], [3, 220, 253], [4, 191, 220], [5, 211, 202], [6, 254, 134], [7, 278, 93], [8, 297, 56], [9, 168, 190], [10, 187, 102], [11, 208, 56], [12, 222, 17], [13, 123, 200], [14, 148, 142], [15, 166, 203], [16, 171, 247], [17, 82, 229], [18, 121, 204], [19, 140, 249], [20, 141, 283]]
fingers :[0, 1, 1, 0, 0]

这个需要结合手图去理解。

其他类似项目手部姿势识别，也都是以此识别出的手指点及坐标为基础，去计算手指角度、距离、是否伸出。来作为业务逻辑的判断前提。

画笔

右⼿⾷指和中指合并时是选择状态，同时弹出颜⾊选框，两指尖移动到对应颜⾊位置时，选中该颜⾊（⿊⾊为橡⽪擦）；⾷指和中指分开始是绘画状态，可在画板上任意绘制。

src/yahboom_esp32_mediapipe/yahboom_esp32_mediapipe/目录下新建文件09_VirtualPaint.py

代码如下：

#!/usr/bin/env python3
# encoding: utf-8
import math
import time
import cv2 as cv
import numpy as np
import mediapipe as mp
import rclpy
from rclpy.node import Node
from cv_bridge import CvBridge
from sensor_msgs.msg import Image, CompressedImage

from rclpy.time import Time
import datetime

xp = yp = pTime = boxx = 0
tipIds = [4, 8, 12, 16, 20]
imgCanvas = np.zeros((480, 640, 3), np.uint8)
brushThickness = 5
eraserThickness = 100
top_height = 50
Color = "Red"
ColorList = {
    'Red': (0, 0, 255),
    'Green': (0, 255, 0),
    'Blue': (255, 0, 0),
    'Yellow': (0, 255, 255),
    'Black': (0, 0, 0),
}

class handDetector:
    def __init__(self, mode=False, maxHands=2, detectorCon=0.5, trackCon=0.5):
        self.tipIds = [4, 8, 12, 16, 20]
        self.mpHand = mp.solutions.hands
        self.mpDraw = mp.solutions.drawing_utils
        #初始化手部检测模型
        self.hands = self.mpHand.Hands(
            static_image_mode=mode,
            max_num_hands=maxHands,
            min_detection_confidence=detectorCon,
            min_tracking_confidence=trackCon )
        self.lmDrawSpec = mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), thickness=-1, circle_radius=15)
        self.drawSpec = mp.solutions.drawing_utils.DrawingSpec(color=(0, 255, 0), thickness=10, circle_radius=10)

    def findHands(self, frame, draw=True):
        self.lmList = []
        img_RGB = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
        self.results = self.hands.process(img_RGB)
        if self.results.multi_hand_landmarks:
            for handLms in self.results.multi_hand_landmarks:
                if draw: self.mpDraw.draw_landmarks(frame, handLms, self.mpHand.HAND_CONNECTIONS, self.lmDrawSpec, self.drawSpec)
                else: self.mpDraw.draw_landmarks(frame, handLms, self.mpHand.HAND_CONNECTIONS)
            for id, lm in enumerate(self.results.multi_hand_landmarks[0].landmark):
                h, w, c = frame.shape
                cx, cy = int(lm.x * w), int(lm.y * h)#节点坐标
                # print(id, cx, cy)  
                self.lmList.append([id, cx, cy])
        return frame, self.lmList
    #手指头漏出检测
    def fingersUp(self):
        fingers=[]
        # Thumb
        if (self.calc_angle(self.tipIds[0],
                            self.tipIds[0] - 1,
                            self.tipIds[0] - 2) > 150.0) and (
                self.calc_angle(
                    self.tipIds[0] - 1,
                    self.tipIds[0] - 2,
                    self.tipIds[0] - 3) > 150.0): fingers.append(1)
        else:
            fingers.append(0)
        # 4 finger
        for id in range(1, 5):
            if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] - 2][2]:
                fingers.append(1)
            else:
                fingers.append(0)
        return fingers


     #欧氏距离
    def get_dist(self, point1, point2):
        x1, y1 = point1
        x2, y2 = point2
        return abs(math.sqrt(math.pow(abs(y1 - y2), 2) + math.pow(abs(x1 - x2), 2)))
    #计算角度
    def calc_angle(self, pt1, pt2, pt3):
        point1 = self.lmList[pt1][1], self.lmList[pt1][2]
        point2 = self.lmList[pt2][1], self.lmList[pt2][2]
        point3 = self.lmList[pt3][1], self.lmList[pt3][2]
        a = self.get_dist(point1, point2)
        b = self.get_dist(point2, point3)
        c = self.get_dist(point1, point3)
        try:#根据余弦定理计算
            radian = math.acos((math.pow(a, 2) + math.pow(b, 2) - math.pow(c, 2)) / (2 * a * b))
            angle = radian / math.pi * 180
        except:
            angle = 0
        return abs(angle)


class MY_Picture(Node):
    def __init__(self, name,ColorROS,ColorListROS):
        super().__init__(name)
        self.bridge = CvBridge()
        self.sub_img = self.create_subscription(
            CompressedImage, '/espRos/esp32camera', self.handleTopic, 1) #获取esp32传来的图像
            
        self.hand_detector = handDetector(detectorCon=0.85)
        self.Color = ColorROS
        self.ColorList = ColorListROS
        self.boxx = 0
        self.xp = 0
        self.yp = 0
        self.top_height = 50
        self.last_stamp = None
        self.new_seconds = 0
        self.fps_seconds = 1

        #回调函数
    def handleTopic(self, msg):
        self.last_stamp = msg.header.stamp  
        if self.last_stamp:
            total_secs = Time(nanoseconds=self.last_stamp.nanosec, seconds=self.last_stamp.sec).nanoseconds
            delta = datetime.timedelta(seconds=total_secs * 1e-9)
            seconds = delta.total_seconds()*100

            if self.new_seconds != 0:
                self.fps_seconds = seconds - self.new_seconds

            self.new_seconds = seconds#保留这次的值
        start = time.time()
        frame = self.bridge.compressed_imgmsg_to_cv2(msg)
        frame = cv.resize(frame, (640, 480))
        h, w, c = frame.shape
        cv.waitKey(10)

        frame,lmList  = self.hand_detector.findHands(frame, draw=False)
        if len(lmList) != 0:
            print("lmList:"+str(lmList))
            # tip of index and middle fingers
            x1, y1 = lmList[8][1:]
            x2, y2 = lmList[12][1:]
            
            fingers = self.hand_detector.fingersUp()
            print(f'fingers :{str(fingers)}')
            if fingers[1] and fingers[2]:#检测到食指与中指
                # print("Seclection mode")
                if y1 < self.top_height:
                    if 0 < x1 < int(w / 5) - 1:
                        self.boxx = 0
                        self.Color = "Red"
                    if int(w / 5) < x1 < int(w * 2 / 5) - 1:
                        self.boxx = int(w / 5)
                        self.Color = "Green"
                    elif int(w * 2 / 5) < x1 < int(w * 3 / 5) - 1:
                        self.boxx = int(w * 2 / 5)
                        self.Color = "Blue"
                    elif int(w * 3 / 5) < x1 < int(w * 4 / 5) - 1:
                        self.boxx = int(w * 3 / 5)
                        self.Color = "Yellow"
                    elif int(w * 4 / 5) < x1 < w - 1:
                        self.boxx = int(w * 4 / 5)
                        self.Color = "Black"#展示颜色选择框
                cv.rectangle(frame, (x1, y1 - 25), (x2, y2 + 25), self.ColorList[self.Color], cv.FILLED)
                cv.rectangle(frame, (self.boxx, 0), (self.boxx + int(w / 5), self.top_height), self.ColorList[self.Color], cv.FILLED)
                cv.rectangle(frame, (0, 0), (int(w / 5) - 1, self.top_height), self.ColorList['Red'], 3)
                cv.rectangle(frame, (int(w / 5) + 2, 0), (int(w * 2 / 5) - 1, self.top_height), self.ColorList['Green'], 3)
                cv.rectangle(frame, (int(w * 2 / 5) + 2, 0), (int(w * 3 / 5) - 1, self.top_height), self.ColorList['Blue'], 3)
                cv.rectangle(frame, (int(w * 3 / 5) + 2, 0), (int(w * 4 / 5) - 1, self.top_height), self.ColorList['Yellow'], 3)
                cv.rectangle(frame, (int(w * 4 / 5) + 2, 0), (w - 1, self.top_height), self.ColorList['Black'], 3)
            if fingers[1] and fingers[2] == False and math.hypot(x2 - x1, y2 - y1) > 50:#检测到食指，没有中指，且食指指尖距离中指指尖>50
                # print("Drawing mode")
                if self.xp == self.yp == 0: self.xp, self.yp = x1, y1
                if self.Color == 'Black':
                    cv.line(frame, (self.xp, self.yp), (x1, y1), self.ColorList[self.Color], eraserThickness)
                    cv.line(imgCanvas, (self.xp, self.yp), (x1, y1), self.ColorList[self.Color], eraserThickness)
                else:
                    cv.line(frame, (self.xp, self.yp), (x1, y1), self.ColorList[self.Color], brushThickness)
                    cv.line(imgCanvas, (self.xp, self.yp), (x1, y1),self.ColorList[self.Color], brushThickness)
                cv.circle(frame, (x1, y1), 15, self.ColorList[self.Color], cv.FILLED)
                self.xp, self.yp = x1, y1
            else: self.xp = self.yp = 0
        imgGray = cv.cvtColor(imgCanvas, cv.COLOR_BGR2GRAY)
        _, imgInv = cv.threshold(imgGray, 50, 255, cv.THRESH_BINARY_INV)
        imgInv = cv.cvtColor(imgInv, cv.COLOR_GRAY2BGR)
        frame = cv.bitwise_and(frame, imgInv)
        frame = cv.bitwise_or(frame, imgCanvas)
        end = time.time()
        fps = 1 / ((end - start)+self.fps_seconds)
        text = "FPS : " + str(int(fps))
        cv.putText(frame, text, (20, 30), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1)
        cv.rectangle(frame, (20, h - 100), (50, h - 70), self.ColorList[self.Color], cv.FILLED)
        cv.putText(frame, self.Color, (70, h - 75), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1)
        cv.imshow('frame', frame)




def main():
    global Color,ColorList
    print("start it")
    rclpy.init()
    esp_img = MY_Picture("My_Picture",Color,ColorList)
    try:
        rclpy.spin(esp_img)
    except KeyboardInterrupt:
        pass
    finally:
        esp_img.destroy_node()
        rclpy.shutdown()

主要处理逻辑，还是获取小车的摄像头画面，转换后调用MediaPipe去做相关的识别后，把手部点及坐标记录下来。在计算判断手指是否伸出。剩下根据业务设计去绘制颜色选择框或者单个食指绘图。

构建后运行：

ros2 run yahboom_esp32_mediapipe VirtualPaint

效果：


http://www.kler.cn/a/519645.html

相关文章：

  • 【架构面试】三、高可用高性能架构设计
  • Gradle自定义任务指南 —— 释放构建脚本的无限可能
  • 解读2025年生物医药创新技术：展览会与论坛的重要性
  • 即梦（Dreamina）技术浅析（一）
  • 自动驾驶中的多传感器时间同步
  • 【自定义函数】编码-查询-匹配
  • python爬虫 爬取站长素材 （图片）(自学6)
  • Pyecharts之词云图、面积图与堆叠面积图
  • 一文讲解Java中的重载、重写及里氏替换原则
  • uniapp商城项目之商品详情
  • 在 Windows 系统上，将 Ubuntu 从 C 盘 迁移到 D 盘
  • 家政预约小程序10首先显示服务内容
  • 有关ORM
  • golang命令大全1--概述
  • Maven面试试题及其答案解析
  • 基础项目实战——学生管理系统（c++）
  • 【Elasticsearch】Springboot编写Elasticsearch的RestAPI
  • Vue 响应式渲染 - 模板语法
  • BroadCom-RDMA博通网卡如何进行驱动安装和设置使得对应网口具有RDMA功能以适配RDMA相机
  • 如何实现一个简单的中文错别字高亮系统？