hand+mic combo

// Tracks points on hands in the webcam. 

// Uses p5.js v1.11.3, and ml5.js v0.20.0 or v1.2.1

// Documentation: https://docs.ml5js.org/#/reference/handpose

//

// As of February 2025, the older ml5 v0.20.0 tracks hands with

// slightly more visual stability than the latest version.

//    Make sure that the following URL is included 

//    in the LIBRARIES field of the SKETCH tab:

//    https://unpkg.com/ml5@0.20.0-alpha.3/dist/ml5.js

//    Use `ml5.handpose` (all lowercase) in preload().

// 

// Uses https://cdn.jsdelivr.net/npm/p5.sound@0.1.0 

// for the mic

​

let myWebcam;

let myHandTracker;

let hands = [];

let trackerOptions = { maxHands: 2, flipHorizontal: true };

let webcamAlpha = 60; // reduce this to make video transparent

let mic; 

let amp; 

let theCurrentVolume = 0; 

​

//----------------------------

function setup() {

  let cnv = createCanvas(640, 480);

  cnv.mousePressed(userStartAudio);

  mic = new p5.AudioIn();

  mic.start();

  amp = new p5.Amplitude();

  mic.disconnect();

  mic.connect(amp);

  initializeWebcamAndHandTracker(); 

}

​

//----------------------------

function draw() {

  background('white');

  // update theCurrentVolume: 

  // Get the overall volume (between 0.0 and 1.0);

  // Smooth the volume variable with a running average

  let v = amp.getLevel();

  let smoothing = 0.50; // must be between 0...1

  let A = smoothing; 

  let B = 1.0-A; 

  theCurrentVolume = (A * theCurrentVolume) + (B * v);

  let diam = map(theCurrentVolume, 0,1, 10,200);

  fill(0); 

  circle(100,100, diam);

  drawWebcamVideo(); 

  drawAllHandPoints(); 

  drawLineBetweenPointerAndThumb(); 

}

​

//----------------------------

function drawAllHandPoints(){

  // Draw all the tracked hand points

  noStroke();

  fill('red');

  textSize(16); 

  textAlign(CENTER);

  if (hands.length == 0){

    // If there are no hands visible

    text("Present hand(s) to camera", width/2, 30); 

  } else {

    for (let i = 0; i < hands.length; i++) {

      let hand = hands[i];

      let keypoints = hand.keypoints; // also available: keypoints3D

      for (let j = 0; j < keypoints.length; j++) {

        let aKeypoint = keypoints[j];

        circle(aKeypoint.x, aKeypoint.y, 10);

        textSize(11); 

        text(j, aKeypoint.x, aKeypoint.y-7);

      }

​

      let whichHand = hand.handedness;

      let wx = keypoints[WRIST].x;

      let wy = keypoints[WRIST].y;

      textSize(20); 

      text(whichHand, wx,wy-50); 

    }

  }

}

​

//----------------------------

function drawLineBetweenPointerAndThumb(){

  // An example showing how to access specific named points.

  // See the index labels at the bottom of this file, below.

  if (hands.length > 0){

    let tx = hands[0].keypoints[THUMB_TIP].x;

    let ty = hands[0].keypoints[THUMB_TIP].y;

    let px = hands[0].keypoints[INDEX_TIP].x;

    let py = hands[0].keypoints[INDEX_TIP].y;

    stroke('black');

    let thick = map(theCurrentVolume, 0,1, 2,200);

    strokeWeight(thick);

    line(tx,ty, px,py);

  }

}

​

//==============================================================

// DON'T CHANGE ANYTHING BELOW THIS LINE

// UNLESS YOU KNOW WHAT YOU ARE DOING. <3

//

//----------------------------

function initializeWebcamAndHandTracker(){

  // Create the webcam video, and start detecting hands:

  myWebcam = createCapture(VIDEO);

  myWebcam.size(640, 480).hide();

  myHandTracker.detectStart(myWebcam, gotHands);

}

​

//----------------------------

function drawWebcamVideo(){

  // Draw the webcam video

  push();

  if (trackerOptions.flipHorizontal){

    translate(myWebcam.width,0); 

    scale(-1,1);

  }

  tint(255,255,255,webcamAlpha); 

  image(myWebcam, 0, 0, myWebcam.width, myWebcam.height);

  pop();

}

​

//----------------------------

function preload() {

  // Load the handpose model. 

  // Note the subtle API difference:

  // Use ml5.handpose for ml5.js v0.20.0

  // Use ml5.handPose for ml5.js v1.2.1

  myHandTracker = ml5.handpose(trackerOptions);

}

​

let avg = 0; 

function gotHands(results) { 

  // If fresh handpose data is received, store it.

  hands = results;

}

​

//----------------------------

// The following hand point index labels may be useful:

const WRIST = 0;

const THUMB_CMC = 1;

const THUMB_MCP = 2;

const THUMB_IP = 3;

const THUMB_TIP = 4;

const INDEX_MCP = 5;

const INDEX_PIP = 6;

const INDEX_DIP = 7;

const INDEX_TIP = 8;

const MIDDLE_MCP = 9;

const MIDDLE_PIP = 10;

const MIDDLE_DIP = 11;

const MIDDLE_TIP = 12;

const RING_MCP = 13;

const RING_PIP = 14;

const RING_DIP = 15;

const RING_TIP = 16;

const PINKY_MCP = 17;

const PINKY_PIP = 18;

const PINKY_DIP = 19;

const PINKY_TIP = 20;