Hand Tracking Mediapipe

let cores = ['#FF5733', '#33FF57', '#3357FF', '#F4D03F', '#9B59B6'];

let indice = 0; // Índice da cor atual

let tempoDeTroca = 10000; // 10 segundos em milissegundos

let ultimoTempo = 0;

var corAtual;

​

//Code copied from Golan Levi

//https://editor.p5js.org/golan/sketches/0yyu6uEwM

​

// p5.js interface to Google MediaPipe Landmark Tracking (Work-in-Progress)

// Allows tracking of landmarks on hands, bodies, and/or faces.

// See https://mediapipe-studio.webapps.google.com/home

// Uses p5.js v.1.8.0 + MediaPipe v.0.10.14

// By Golan Levin, version of 10/29/2023

// Huge thanks to Orr Kislev, who made it work in p5's global mode! Thanks Orr!

// His fix: https://editor.p5js.org/orrkislev/sketches/wwLqrnVDt

​

// Don't change the names of these global variables.

let myHandLandmarker;

let myPoseLandmarker;

let myFaceLandmarker;

let handLandmarks;

let poseLandmarks;

let faceLandmarks;

let myCapture;

let lastVideoTime = -1;

​

// For landmarks you want, set to true; set false the ones you don't.

// Works best with just one or two sets of landmarks.

const trackingConfig = {

  doAcquireHandLandmarks: true,

  doAcquirePoseLandmarks: !true,

  doAcquireFaceLandmarks: !true,

  doAcquireFaceMetrics: !true,

  poseModelLiteOrFull: "lite", /* "lite" (3MB) or "full" (6MB) */

  cpuOrGpuString: "GPU", /* "GPU" or "CPU" */

  maxNumHands: 2,

  maxNumPoses: 1,

  maxNumFaces: 1,

};

​

// Notes & Observations:

// • Works best with hands OR body OR face — but lags when more than one is enabled.

// • Seems to slow down after a while

// • ToDo: add getters for special named points; see:

// • Don't forget to disable MacOS camera "Reactions" (in Sonoma+).

​

//------------------------------------------

async function preload() {

  const mediapipe_module = await import('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/vision_bundle.js');

  HandLandmarker = mediapipe_module.HandLandmarker;

  PoseLandmarker = mediapipe_module.PoseLandmarker;

  FaceLandmarker = mediapipe_module.FaceLandmarker;

  FilesetResolver = mediapipe_module.FilesetResolver;

  const vision = await FilesetResolver.forVisionTasks(

    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm"

  );

​

  // Hand Landmark Tracking:

  // https://codepen.io/mediapipe-preview/pen/gOKBGPN

  // https://mediapipe-studio.webapps.google.com/studio/demo/hand_landmarker

  if (trackingConfig.doAcquireHandLandmarks){

    myHandLandmarker = await HandLandmarker.createFromOptions(vision, {

      numHands: trackingConfig.maxNumHands,

      runningMode: "VIDEO",

      baseOptions: {

        delegate: trackingConfig.cpuOrGpuString,

        modelAssetPath:

          "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",

      },

    });

  }

  // Pose (Body) Landmark Tracking: 

  // https://codepen.io/mediapipe-preview/pen/abRLMxN

  // https://developers.google.com/mediapipe/solutions/vision/pose_landmarker

  if (trackingConfig.doAcquirePoseLandmarks){

    const poseModelLite = "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task";

    const poseModelFull = "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task";

    let poseModel = poseModelLite;

    poseModel = (trackingConfig.poseModelLiteOrFull == "full") ? poseModelFull:poseModelLite;

    myPoseLandmarker = await PoseLandmarker.createFromOptions(vision, {

      numPoses: trackingConfig.maxNumPoses,

      runningMode: "VIDEO",

      baseOptions: {

        modelAssetPath: poseModel,

        delegate:trackingConfig.cpuOrGpuString,

      },

    });

  }

  // Face Landmark Tracking:

  // https://codepen.io/mediapipe-preview/pen/OJBVQJm

  // https://developers.google.com/mediapipe/solutions/vision/face_landmarker

  if (trackingConfig.doAcquireFaceLandmarks){

    myFaceLandmarker = await FaceLandmarker.createFromOptions(vision, {

      numFaces: trackingConfig.maxNumFaces,

      runningMode: "VIDEO",

      outputFaceBlendshapes:trackingConfig.doAcquireFaceMetrics,

      baseOptions: {

        delegate: trackingConfig.cpuOrGpuString,

        modelAssetPath:

          "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",

      },

    });

  }

}

​

//------------------------------------------

async function predictWebcam() {

  let startTimeMs = performance.now();

  if (lastVideoTime !== myCapture.elt.currentTime) {

    if (trackingConfig.doAcquireHandLandmarks && myHandLandmarker) {

      handLandmarks = myHandLandmarker.detectForVideo(myCapture.elt,startTimeMs);

    }

    if (trackingConfig.doAcquirePoseLandmarks && myPoseLandmarker) {

      poseLandmarks = myPoseLandmarker.detectForVideo(myCapture.elt,startTimeMs);

    }

    if (trackingConfig.doAcquireFaceLandmarks && myFaceLandmarker) {

      faceLandmarks = myFaceLandmarker.detectForVideo(myCapture.elt,startTimeMs);

    }

    lastVideoTime = myCapture.elt.currentTime;

  }

  window.requestAnimationFrame(predictWebcam);

}

​

​

//------------------------------------------

function setup() {

  createCanvas(640, 480);

  myCapture = createCapture(VIDEO);

  myCapture.size(320, 240);

  myCapture.hide();

  alterarCor(); // Muda a cor inicial

}

​

function draw() {

  background("white");

  predictWebcam();

  drawVideoBackground();

  drawHandPoints();

  drawPosePoints(); 

  drawFacePoints();

  drawFaceMetrics();

  drawDiagnosticInfo();

  // Verifica se já passou o tempo de 10 segundos

   let agora = millis();

  if (agora - ultimoTempo >= tempoDeTroca) {

    indice++; // Avança para a próxima cor

    if (indice >= cores.length) {

      noLoop(); // Para o loop quando acabar a lista de cores

      console.log("Fim da lista de cores.");

    } else {

      alterarCor(); // Muda a cor

    }

    ultimoTempo = agora; // Atualiza o tempo de referência

  }

}

​

function alterarCor() {

  corAtual = cores[indice]

}

​

​

//------------------------------------------

function drawVideoBackground() {

  push();

  translate(width, 0);

  scale(-1, 1);

  tint(255, 255, 255, 72);

  image(myCapture, 0, 0, width, height);

  tint(255);

  pop();

}

​

//------------------------------------------

// HANDS: 21 2D landmarks per hand, up to maxNumHands at once

function drawHandPoints() {

  if (trackingConfig.doAcquireHandLandmarks) {

    if (handLandmarks && handLandmarks.landmarks) {

      const nHands = handLandmarks.landmarks.length;

      if (nHands > 0) {

        // Draw lines connecting the joints of the fingers

        noFill();

        stroke("black");

        strokeWeight(2.0);

        for (let h = 0; h < nHands; h++) {

          let joints = handLandmarks.landmarks[h];

          drawConnectors(joints, HANDLANDMARKER_PALM);

          drawConnectors(joints, HANDLANDMARKER_THUMB);

          drawConnectors(joints, HANDLANDMARKER_INDEX_FINGER);

          drawConnectors(joints, HANDLANDMARKER_MIDDLE_FINGER);

          drawConnectors(joints, HANDLANDMARKER_RING_FINGER);

          drawConnectors(joints, HANDLANDMARKER_PINKY);

        }

        // Draw just the joints of the hands

        strokeWeight(1.0);

        stroke("black");

        fill("red");

        for (let h = 0; h < nHands; h++) {

          let joints = handLandmarks.landmarks[h];

          let handX = joints.x;

          let handY = joints.y;

          let handZ = joints.z;

​

          // Desenhar pontos das mãos no canvas

          fill(corAtual);

          noStroke();

          ellipse(handX * width, handY * height, 10, 10);  // Desenha círculos nas posições das mãos

​

          for (let i = 0; i <= 20; i++) {

            let px = joints[i].x;

            let py = joints[i].y;

            px = map(px, 0, 1, width, 0);

            py = map(py, 0, 1, 0, height);

            circle(px, py, 9);

          }

        }

      }

    }

  }

}

​

//------------------------------------------

// 33 joints of the body. See landmarks.js for the list. 

function drawPosePoints(){

  if (trackingConfig.doAcquirePoseLandmarks) {

    if (poseLandmarks && poseLandmarks.landmarks) {

      const nPoses = poseLandmarks.landmarks.length;

      if (nPoses > 0) {

        // Draw lines connecting the joints of the body

        noFill();

        stroke("darkblue");

        strokeWeight(2.0);

        for (let h = 0; h < nPoses; h++) {

          let joints = poseLandmarks.landmarks[h];

          drawConnectors(joints, PoseLandmarker.POSE_CONNECTIONS);

        }

      }

    }

  }

}

​

​

//------------------------------------------

// Tracks 478 points on the face. 

function drawFacePoints() {

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

            let nFaceLandmarks = aFace.length;

            noFill();

            stroke("black");

            strokeWeight(1.0);

            for (let i = 0; i < nFaceLandmarks; i++) {

              let px = aFace[i].x;

              let py = aFace[i].y;

              px = map(px, 0, 1, width, 0);

              py = map(py, 0, 1, 0, height);

              circle(px, py, 1);

            }

            noFill();

            stroke("black");

            strokeWeight(2.0);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_EYE);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_FACE_OVAL);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LIPS);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS);

            drawConnectors(aFace, FACELANDMARKER_NOSE); // Google offers no nose

          }

        }

      }

    }

  }

}

​

function drawFaceMetrics(){

  if (trackingConfig.doAcquireFaceLandmarks && 

      trackingConfig.doAcquireFaceMetrics){

    if (faceLandmarks && faceLandmarks.faceBlendshapes) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      for (let f = 0; f < nFaces; f++) {

        let aFaceMetrics = faceLandmarks.faceBlendshapes[f];

        if (aFaceMetrics){

          fill('black'); 

          textSize(7); 

          let tx = 40; 

          let ty = 40; 

          let dy = 8.5;

          let vx0 = tx-5; 

          let vx1 = tx-35;

          let nMetrics = aFaceMetrics.categories.length; 

          for (let i=1; i<nMetrics; i++){

            let metricName = aFaceMetrics.categories[i].categoryName;

            noStroke();

            text(metricName, tx,ty); 

            let metricValue = aFaceMetrics.categories[i].score;

            let vx = map(metricValue,0,1,vx0,vx1);

            stroke(0,0,0); 

            strokeWeight(2.0); 

            line(vx0,ty-2, vx,ty-2); 

            stroke(0,0,0,20);

            line(vx0,ty-2, vx1,ty-2); 

            ty+=dy;

          }

        }

      }

    }

  }

}

​

​

//------------------------------------------

function drawConnectors(landmarks, connectorSet) {

  if (landmarks) {

    let nConnectors = connectorSet.length;

    for (let i=0; i<nConnectors; i++){

      let index0 = connectorSet[i].start; 

      let index1 = connectorSet[i].end;

      let x0 = map(landmarks[index0].x, 0,1, width,0);

      let y0 = map(landmarks[index0].y, 0,1, 0,height);

      let x1 = map(landmarks[index1].x, 0,1, width,0);

      let y1 = map(landmarks[index1].y, 0,1, 0,height);

      line(x0,y0, x1,y1); 

    }

  }

}

​

//------------------------------------------

function drawDiagnosticInfo() {

  noStroke();

  fill("black");

  textSize(12); 

  text("FPS: " + int(frameRate()), 40, 30);

}