M Beat Box

let video;

let poseNet;

let poses = [];

let averageConfidence = .2

let recentPoses = [];

const maxPoseArrayLength = 10;

let startingPose = false;

let screenX = 640;

let screenY = 480;

let yellowGuideDotX = screenX * 0.351;

let yellowGuideDotY = screenY / 1.8;

let blueGuideDotX = screenX * 0.64;

let blueGuideDotY = screenY / 1.8;

​

​

// Sounds

​

let melodyOne;

let melodyTwo;

let snare;

let snareTwo;

let brightkick;

​

function setup() {

    createCanvas(screenX, screenY);

    video = createCapture(VIDEO);

    video.size(width, height);

    poseNet = ml5.poseNet(video, modelReady);

    poseNet.on('pose', function (results) {

        poses = results;

    });

    video.hide();

    melodyOne = loadSound('melodyOne.wav');

    melodyTwo = loadSound('melodyTwo.wav');

    snare = loadSound('snare.wav');

    snareTwo = loadSound('snareTwo.wav');

    brightkick = loadSound('brightkick.wav');

    redMan4 = loadImage('redMan4.png');

}

​

​

function modelReady() {

    console.log("modelReady")

}

​

function draw() {

​

​

    background(120, 100, 80);

    videoMirror();

    redMan();

    drawWrists();

    melodySounds();

    drumSounds();

    noteLines();

}

​

function videoMirror() {

    translate(width, 0);

    scale(-1, 1);

    image(video, 0, 0, width, height);

}

​

function getBestPose(candidateResults) {

    let currentBestResult = candidateResults[0];

    if (currentBestResult === undefined) {

        return

    }

    candidateResults.forEach(candidate => {

        if (candidate.pose.score > currentBestResult.pose.score) {

            currentBestResult = candidate;

        }

    });

    return currentBestResult.pose;

}

​

function getWristPositionAverageRightWrist() {

    let averageRightX = 0;

    let averageRightY = 0;

​

    recentPoses.forEach(recentP => {

        averageRightX += recentP.rightWrist.x

        averageRightY += recentP.rightWrist.y

    })

    averageRightX = averageRightX / recentPoses.length

    averageRightY = averageRightY / recentPoses.length

    return { rightX: averageRightX, rightY: averageRightY }

}

​

function getWristPositionAverageLeftWrist() {

    let averageLeftX = 0;

    let averageLeftY = 0;

​

    recentPoses.forEach(recentP => {

        averageLeftX += recentP.leftWrist.x

        averageLeftY += recentP.leftWrist.y

    })

    averageLeftX = averageLeftX / recentPoses.length

    averageLeftY = averageLeftY / recentPoses.length

    return { leftX: averageLeftX, leftY: averageLeftY }

}

​

// silhouette for starting

​

function redMan() {

    if (startingPose === true) {

        return;

    }

    image(redMan4, width / 4, height / 7, 0, 0)

    fill(200, 200, 0)

    ellipse(yellowGuideDotX, yellowGuideDotY, 20);

    fill(0, 200, 200)

    ellipse(blueGuideDotX, blueGuideDotY, 20);

    const wristPointLeft = getWristPositionAverageLeftWrist();

    const wristPointRight = getWristPositionAverageRightWrist();

​

    let distanceBetweenYellowDots = (dist(yellowGuideDotX, yellowGuideDotY, wristPointRight.rightX, wristPointRight.rightY));

    let distanceBetweenBlueDots = (dist(blueGuideDotX, blueGuideDotY, wristPointLeft.leftX, wristPointLeft.leftY));

    if (distanceBetweenYellowDots < 10 && distanceBetweenBlueDots < 10) {

        startingPose = true

    }

}

​

// draws wrist dots for interface

​

function drawWrists() {

    if (poses.length === 0) {

        return;

    }

    const bestPose = getBestPose(poses);

    const wristPointLeft = getWristPositionAverageLeftWrist();

    const wristPointRight = getWristPositionAverageRightWrist();

​

    if (recentPoses.length > maxPoseArrayLength) {

        recentPoses.shift();

    }

    if (bestPose

        && bestPose.leftWrist

        && bestPose.rightWrist) {

        // && bestPose.leftWrist.confidence > averageConfidence){

​

        recentPoses.push(bestPose);

        ellipseMode(CENTER);

        noStroke();

        fill(0, 200, 200);

        ellipse(wristPointLeft.leftX, wristPointLeft.leftY, 30);

        // console.log(bestPose);

    }

    if (bestPose

        && bestPose.leftWrist

        && bestPose.rightWrist) {

        // && bestPose.RightWrist.confidence > averageConfidence ) {

​

        ellipseMode(CENTER);

        noStroke();

        fill(200, 200, 0);

        ellipse(wristPointRight.rightX, wristPointRight.rightY, 30);

    }

}

​

function melodySounds() {

    const wristPointLeft = getWristPositionAverageLeftWrist();

    if (startingPose === true

        && wristPointLeft.leftY <= height / 3

        && !melodyOne.isPlaying()) {

        melodyOne.play();

        tint(233, 0, 0, 200);

    }

}

​

// guide to where the note will play when wrist dot crosses 

​

function noteLines() {

    if (startingPose === true) {

        stroke(243, 244, 0)

        line(0, height / 3, width * 0.4, height / 3)

        stroke(233, 0, 0)

        line(width * 0.6, height / 3, width, height / 3)

        stroke(0, 243, 244)

        line(0, height * 0.7, width * 0.4, height * 0.7)

        stroke(243, 0, 244)

        line(width * 0.6, height * 0.7, width, height * 0.7)

    }

}

​

function drumSounds() {

    const wristPointRight = getWristPositionAverageRightWrist();

    const wristPointLeft = getWristPositionAverageLeftWrist();

    if (startingPose === true

        && wristPointRight.rightY <= height / 3

        && !snare.isPlaying()) {

        snare.play();

        tint(243, 244, 0, 200);

    }

    if (startingPose === true

        && wristPointRight.rightY >= height * 0.7

        && !brightkick.isPlaying()) {

        brightkick.play();

        tint(0, 243, 244, 200);

    }

    if (startingPose === true

        && wristPointLeft.leftY >= height * 0.7

        && !snareTwo.isPlaying()) {

        snareTwo.play();

        tint(243, 0, 244, 200);

    }

}

​

​

​