Webcam Trails

const model = bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation;

const segmenterConfig = {

  runtime: 'tfjs',

  solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@1.0.2'

                // or 'base/node_modules/@mediapipe/selfie_segmentation' in npm.

};

const segmentationConfig = { flipHorizontal: false, landscape: true };

​

let segmenter

let capture

let running

​

let maskShader

let bgPicker

let picker

let video

let start

​

let fbo

​

function maskShaderSource() {

  const vert = `

    precision highp float;

    precision highp int;

​

    attribute vec3 aPosition;

    attribute vec2 aTexCoord;

    attribute vec4 aVertexColor;

​

    uniform mat4 uModelViewMatrix;

    uniform mat4 uProjectionMatrix;

​

    varying vec2 vTexCoord;

​

    void main(void) {

      vec4 viewModelPosition = uModelViewMatrix * vec4(aPosition, 1.0);

      gl_Position = uProjectionMatrix * viewModelPosition;  

      vTexCoord = aTexCoord;

    }

  `

​

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D content;

    uniform sampler2D mask;

​

    varying vec2 vTexCoord;

​

    void main(void) {

      gl_FragColor = vec4(texture2D(content, vTexCoord).xyz, 1.) * smoothstep(0.3, 0.9, texture2D(mask, vTexCoord).x);

    }

  `

​

  return [vert, frag]

}

​

let cam

let bg

let fg

let captureData

let maskData

let prevMaskData

let fboSize

​

async function setup() {

  createCanvas(800, 800, WEBGL)

  pixelDensity(1)

  const isMobile = window.navigator.userAgent && /Mobi|Android/i.test(window.navigator.userAgent)

  // cam = createCapture(VIDEO, { flipped: true })

  cam = createCapture(isMobile ? {

    audio: false,

    video: {

      facingMode: {

        exact: "environment"

      }

    },

  } : VIDEO, { flipped: !isMobile })

  cam.hide()

  fboSize = { width: 800, height: 800 }

  bg = createFramebuffer()

  fg = createFramebuffer()

  captureData = createGraphics(fboSize.width, fboSize.height)

  maskData = createGraphics(fboSize.width, fboSize.height)

  prevMaskData = createGraphics(fboSize.width, fboSize.height)

  fbo = createFramebuffer(fboSize)

  maskShader = createShader(...maskShaderSource())

  segmenter = await bodySegmentation.createSegmenter(model, segmenterConfig)

}

​

let didRun = false

​

function draw() {

  if (!segmenter) return

  if (!running) {

    running = true

    captureData.clear()

    captureData.image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER)

    let mask

    segmenter.segmentPeople(captureData.elt, segmentationConfig).then((res) => {

      mask = res[0].mask.mask

      return res[0].mask.toImageData()

    }).then((img) => {

      prevMaskData.clear()

      prevMaskData.image(maskData, 0, 0)

      maskData.clear()

      maskData.drawingContext.putImageData(img, 0, 0)

      maskData.filter(BLUR, 2)

      maskData.push()

      // maskData.blendMode(ADD)

      // maskData.tint(255, 180)

      // maskData.image(prevMaskData, 0, 0)

      maskData.pop()

      // maskData.filter(ERODE)

      // maskData.filter(BLUR, 2)

      fbo.begin()

      clear()

      shader(maskShader)

      maskShader.setUniform('content', captureData)

      maskShader.setUniform('mask', maskData)

      noStroke()

      plane(fboSize.width, fboSize.height)

      fbo.end()

      mask.dispose()

      running = false

    })

  }

  imageMode(CENTER)

  bg.draw(() => image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER))

  fg.draw(() => image(fbo, 0, 0))

  image(bg, 0, 0)

  image(fg, 0, 0)

}

​

function mouseClicked() {

  save('trails.png')

}