Facial Expression Decoration

// Using the libraray of face-api.js https://github.com/justadudewhohacks/face-api.js/

// The program reads the expression (emotion) and eye position detected by face-api, and draw emoji and additional expression accordingly

​

var capture;

var result;

var showMarks = false;

var expressionNames = ['neutral', 'happy', 'sad', 'angry', 'surprised'];

var emojis = ['?', '?', '?', '?', '?'];

var shockedEye;

var heartLeftEye;

var angryLeftEye;

var tearLeftEye;

var heartRightEye;

var angryRightEye;

var tearRightEye;

​

function setup() {

  createCanvas(640, 680);

  // refer to https://p5js.org/reference/#/p5/createCapture

  capture = createCapture(VIDEO, async () => {

    // read machine learning models - refer to the library api doc https://github.com/justadudewhohacks/face-api.js/

    await faceapi.loadSsdMobilenetv1Model('./');

    await faceapi.loadFaceLandmarkModel('./');

    await faceapi.loadFaceRecognitionModel('./');

    await faceapi.loadFaceExpressionModel('./');

    getExpression();

  });

  capture.size(640, 480);

  shockedEye = loadImage('surprise.png');

  heartLeftEye = loadImage('happy.png');

  angryLeftEye = loadImage('angry.png');

  tearLeftEye = loadImage('tear.png');

  heartRightEye = loadImage('happy_right.png');

  angryRightEye = loadImage('angry_right.png');

  tearRightEye = loadImage('tear_right.png');

}

​

async function getExpression(){

  result = await faceapi.detectSingleFace(capture.elt).withFaceLandmarks().withFaceExpressions();

  // make the function call itself to repeat forever

  getExpression();

}

​

function draw() {

  background(0);

  strokeWeight(0);

  fill(255, 100, 100);

  textSize(30);

  if (!result) {

    text('Wait for successful face detection...', 50, 550);

  }

  else {

    console.log(result.landmarks.getLeftEye());

    console.log(result);

    image(capture, 0, 0, 640, 480);

    text('Press m to show/hide landmarks', 50, 550);

    if (showMarks) {

      drawMarks();

    }

    var expressionNumber = detectExpression();

    // draw emoji text

    textSize(50);

    text(emojis[expressionNumber], 30, 70);

    // calculate eye center from api returned eye marks

    var leftEyeCenter = eyeCenter(result.landmarks.getLeftEye());

    var rightEyeCenter = eyeCenter(result.landmarks.getRightEye());

    // happy

    if (expressionNumber == 1) {

      image(heartLeftEye, leftEyeCenter._x - 25, leftEyeCenter._y - 25, 50, 50);

      image(heartRightEye, rightEyeCenter._x - 25, rightEyeCenter._y - 25, 50, 50);

    }

    // sad

    if (expressionNumber == 2) {

      image(tearLeftEye, leftEyeCenter._x - 30, leftEyeCenter._y, 40, 100);

      image(tearRightEye, rightEyeCenter._x - 10, rightEyeCenter._y, 40, 100);

    }

    // angry

    if (expressionNumber == 3) {

      image(angryLeftEye, leftEyeCenter._x - 125, leftEyeCenter._y - 40, 80, 80);

      image(angryRightEye, rightEyeCenter._x + 45, rightEyeCenter._y - 40, 80, 80);

    }

    // surprised

    if (expressionNumber == 4) {

      image(shockedEye, leftEyeCenter._x - 210, leftEyeCenter._y - 100, 240, 144);

      image(shockedEye, rightEyeCenter._x - 210, rightEyeCenter._y - 100, 240, 144);

    }

  }

}

​

// draw landmarks over the face

function drawMarks() {

  stroke(255, 100, 100);

  strokeWeight(4);

  for (var i = 0; i < result.landmarks._positions.length; i++) {

    var mark = result.landmarks._positions[i];

    point(mark._x, mark._y);

  }

}

​

function detectExpression() {

  var probability = 0.0;

  // the most possible expression number from 0 - 4

  var expressionNumber = 0;

  for (var i = 0; i < 5; i++) {

    if (result.expressions[expressionNames[i]] > probability) {

      probability = result.expressions[expressionNames[i]];

      expressionNumber = i;

    }

  }

  return expressionNumber;

}

​

function eyeCenter(eyeMarks) {

  var centerX = 0;

  var centerY = 0;

  for (var i = 0; i < eyeMarks.length; i++) {

    centerX += eyeMarks[i]._x;

    centerY += eyeMarks[i]._y;

  }

  centerX /= eyeMarks.length;

  centerY /= eyeMarks.length;

  return {_x: centerX, _y: centerY};

}

​

function keyPressed() {

  if (key == 'm') {

    showMarks = !showMarks;

  }

}