FaceExtractor-2024

// p5.js interface to Google MediaPipe Face Tracking 

// This demo extracts textured triangles from the face.

// See https://mediapipe-studio.webapps.google.com/home

// Uses p5.js v.1.11.0 + MediaPipe v.0.10.17

// By Golan Levin, version of 10/28/2024

// Click to toggle nose effect. 

​

// Global variables.

let doNoseFun = true; 

let myFaceLandmarker;

let faceLandmarks;

let myCapture;

let lastVideoTime = -1;

​

const trackingConfig = {

  doAcquireFaceLandmarks: true,

  doAcquireFaceMetrics: true,

  cpuOrGpuString: "GPU", /* "GPU" or "CPU" */

  maxNumFaces: 1,

};

​

​

//------------------------------------------

async function preload() {

  const mediapipe_module = await import('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/vision_bundle.js');

  FaceLandmarker = mediapipe_module.FaceLandmarker;

  FilesetResolver = mediapipe_module.FilesetResolver;

  const vision = await FilesetResolver.forVisionTasks(

    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17/wasm"

  );

  // Face Landmark Tracking:

  // https://codepen.io/mediapipe-preview/pen/OJBVQJm

  // https://developers.google.com/mediapipe/solutions/vision/face_landmarker

  if (trackingConfig.doAcquireFaceLandmarks){

    myFaceLandmarker = await FaceLandmarker.createFromOptions(vision, {

      numFaces: trackingConfig.maxNumFaces,

      runningMode: "VIDEO",

      outputFaceBlendshapes:trackingConfig.doAcquireFaceMetrics,

      baseOptions: {

        delegate: trackingConfig.cpuOrGpuString,

        modelAssetPath:

          "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",

      },

    });

  }

}

​

//------------------------------------------

async function predictWebcam() {

  let startTimeMs = performance.now();

  if (lastVideoTime !== myCapture.elt.currentTime) {

    if (trackingConfig.doAcquireFaceLandmarks && myFaceLandmarker) {

      faceLandmarks = myFaceLandmarker.detectForVideo(myCapture.elt,startTimeMs);

    }

    lastVideoTime = myCapture.elt.currentTime;

  }

  window.requestAnimationFrame(predictWebcam);

}

​

​

//------------------------------------------

function setup() {

  createCanvas(640, 480, WEBGL);

  myCapture = createCapture(VIDEO);

  myCapture.size(640, 480);

  myCapture.hide();

}

​

function keyPressed(){

  doNoseFun = !doNoseFun; 

}

​

function draw() {

  background("white");

  push(); 

  translate(-width/2, -height/2); 

  predictWebcam();

  drawVideoBackground();

  // drawExtractedFaceRect();

  drawFaceTrianglesTextured(); 

  // drawFaceConnectors(); 

  // drawFaceTriangles(); 

  // drawFacePoints();

  // drawFaceMetrics(); 

  pop(); 

}

​

​

//------------------------------------------

function drawVideoBackground() {

  push();

  translate(width, 0);

  scale(-1, 1);

  tint(255, 255, 255, 60);

  image(myCapture, 0, 0, width, height);

  tint(255);

  pop();

}

​

​

//------------------------------------------

function drawExtractedFaceRect(){

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

            let nFaceLandmarks = aFace.length;

            let minx = 9999999;

            let maxx = -9999999;

            let miny = 9999999;

            let maxy = -9999999;

            for (let i = 0; i < nFaceLandmarks; i++) {

              let px = aFace[i].x;

              let py = aFace[i].y;

              px = map(px, 0, 1, width, 0);

              py = map(py, 0, 1, 0, height);

              if (px > maxx){ maxx = px; }

              if (py > maxy){ maxy = py; }

              if (px < minx){ minx = px; }

              if (py < miny){ miny = py; }

            }

            let vx = width-minx;

            let vy = miny;

            let vw = (width-minx)-(width-maxx);

            vx -= vw;

            let vh = maxy-miny;

            let subImage = myCapture.get(vx,vy, vw,vh);

            push();

            translate(width-vx,0);

            scale(-1,1);

            image(subImage, 0,vy,vw,vh);

            pop();

          }

        }

      }

    }

  }

}

​

//------------------------------------------

// Tracks 478 points on the face. 

function drawFacePoints() {

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

            noFill();

            stroke("black");

            strokeWeight(1.0);

            let nFaceLandmarks = aFace.length;

            for (let i = 0; i < nFaceLandmarks; i++) {

              let px = aFace[i].x;

              let py = aFace[i].y;

              px = map(px, 0, 1, width, 0);

              py = map(py, 0, 1, 0, height);

              circle(px, py, 1);

            }

          }

        }

      }

    }

  }

}

​

//------------------------------------------

function drawFaceConnectors(){

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

            noFill();

            stroke("black");

            strokeWeight(2.0);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_EYE);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_FACE_OVAL);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LIPS);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS);

            drawConnectors(aFace, FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS);

            drawConnectors(aFace, FACELANDMARKER_NOSE); // Google offers no nose

          }

        }

      }

    }

  }

}

​

//------------------------------------------

function drawFaceTriangles(){

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

​

            stroke('black');

            strokeWeight(0.5); 

            noFill(); 

            let triSet = TRI468;

            let vtxSet = VTX468;

            for (let i=0; i<(triSet.length); i+=3){

              let i0=vtxSet[triSet[i+0]];

              let i1=vtxSet[triSet[i+1]];

              let i2=vtxSet[triSet[i+2]];

              let x0 = map(aFace[i0].x, 0,1, width,0);

              let y0 = map(aFace[i0].y, 0,1, 0,height);

              let x1 = map(aFace[i1].x, 0,1, width,0);

              let y1 = map(aFace[i1].y, 0,1, 0,height);

              let x2 = map(aFace[i2].x, 0,1, width,0);

              let y2 = map(aFace[i2].y, 0,1, 0,height);

              triangle(x0,y0, x1,y1, x2,y2); 

            }

          }

        }

      }

    }

  }

}

​

​

//------------------------------------------

function drawFaceTrianglesTextured(){

  if (trackingConfig.doAcquireFaceLandmarks) {

    if (faceLandmarks && faceLandmarks.faceLandmarks) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      if (nFaces > 0) {

        for (let f = 0; f < nFaces; f++) {

          let aFace = faceLandmarks.faceLandmarks[f];

          if (aFace) {

            let noseIndex = 4; 

            let noseMag = 30;

            let triSet = TRI468;

            let vtxSet = VTX468;

            noStroke(); 

            tint("white")

            texture(myCapture);

            textureMode(NORMAL);

            beginShape(TRIANGLES);

            for (let i=0; i<(triSet.length); i+=3){

              // Fetch vertex indices for each triangle

              let i0=vtxSet[triSet[i+0]];

              let i1=vtxSet[triSet[i+1]];

              let i2=vtxSet[triSet[i+2]];

              // Texture coordinates (UV's, 0..1)

              let t0x = aFace[i0].x;

              let t0y = aFace[i0].y;

              let t1x = aFace[i1].x;

              let t1y = aFace[i1].y;

              let t2x = aFace[i2].x;

              let t2y = aFace[i2].y;

              // Mesh coordinates (screenspace)

              let x0 = map(t0x, 0,1, width,0);

              let y0 = map(t0y, 0,1, 0,height);

              let x1 = map(t1x, 0,1, width,0);

              let y1 = map(t1y, 0,1, 0,height);

              let x2 = map(t2x, 0,1, width,0);

              let y2 = map(t2y, 0,1, 0,height);

              // Warp mesh coordinates near the nose.  

              if (doNoseFun){

                let mx = (1-aFace[noseIndex].x)*width;

                let my = aFace[noseIndex].y*height;

                let dx0 = x0-mx; 

                let dy0 = y0-my; 

                let dh0 = sqrt(dx0*dx0+dy0*dy0);

                let dx1 = x1-mx; 

                let dy1 = y1-my;

                let dh1 = sqrt(dx1*dx1+dy1*dy1);

                let dx2 = x2-mx; 

                let dy2 = y2-my;

                let dh2 = sqrt(dx2*dx2+dy2*dy2);

                if (dh0 > 0) {x0 += noseMag * dx0/dh0; y0 += noseMag * dy0/dh0;}

                if (dh1 > 0) {x1 += noseMag * dx1/dh1; y1 += noseMag * dy1/dh1;}

                if (dh2 > 0) {x2 += noseMag * dx2/dh2; y2 += noseMag * dy2/dh2;}

              }

              vertex(x0,y0, t0x,t0y); 

              vertex(x1,y1, t1x,t1y); 

              vertex(x2,y2, t2x,t2y); 

            }

            endShape(); 

          }

        }

      }

    }

  }

}

​

​

//------------------------------------------

function drawFaceMetrics(){

  if (trackingConfig.doAcquireFaceLandmarks && 

      trackingConfig.doAcquireFaceMetrics){

    if (faceLandmarks && faceLandmarks.faceBlendshapes) {

      const nFaces = faceLandmarks.faceLandmarks.length;

      for (let f = 0; f < nFaces; f++) {

        let aFaceMetrics = faceLandmarks.faceBlendshapes[f];

        if (aFaceMetrics){

          fill('black'); 

          textSize(7); 

          let tx = 40; 

          let ty = 40; 

          let dy = 8.5;

          let vx0 = tx-5; 

          let vx1 = tx-35;

          let nMetrics = aFaceMetrics.categories.length; 

          for (let i=1; i<nMetrics; i++){

            let metricName = aFaceMetrics.categories[i].categoryName;

            noStroke();

            text(metricName, tx,ty); 

            let metricValue = aFaceMetrics.categories[i].score;

            let vx = map(metricValue,0,1,vx0,vx1);

            stroke(0,0,0); 

            strokeWeight(2.0); 

            line(vx0,ty-2, vx,ty-2); 

            stroke(0,0,0,20);

            line(vx0,ty-2, vx1,ty-2); 

            ty+=dy;

          }

        }

      }

    }

  }

}

​

​

//------------------------------------------

function drawConnectors(landmarks, connectorSet) {

  if (landmarks) {

    let nConnectors = connectorSet.length;

    for (let i=0; i<nConnectors; i++){

      let index0 = connectorSet[i].start; 

      let index1 = connectorSet[i].end;

      let x0 = map(landmarks[index0].x, 0,1, width,0);

      let y0 = map(landmarks[index0].y, 0,1, 0,height);

      let x1 = map(landmarks[index1].x, 0,1, width,0);

      let y1 = map(landmarks[index1].y, 0,1, 0,height);

      line(x0,y0, x1,y1); 

    }

  }

}

​

//------------------------------------------

function drawDiagnosticInfo() {

  noStroke();

  fill("black");

  textSize(12); 

  text("FPS: " + int(frameRate()), 40, 30);

}