Webcam Warp

const model = bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation;

const segmenterConfig = {

  runtime: 'tfjs',

  solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@1.0.2'

                // or 'base/node_modules/@mediapipe/selfie_segmentation' in npm.

};

const segmentationConfig = { flipHorizontal: false, landscape: true };

​

let font

​

let segmenter

let capture

let running

​

let maskShader

let warpShader

let bgPicker

let picker

let video

let start

​

let fbo

​

const vert = `

  precision highp float;

  precision highp int;

​

  attribute vec3 aPosition;

  attribute vec2 aTexCoord;

  attribute vec4 aVertexColor;

​

  uniform mat4 uModelViewMatrix;

  uniform mat4 uProjectionMatrix;

​

  varying vec2 vTexCoord;

​

  void main(void) {

    vec4 viewModelPosition = uModelViewMatrix * vec4(aPosition, 1.0);

    gl_Position = uProjectionMatrix * viewModelPosition;  

    vTexCoord = aTexCoord;

  }

`

​

function maskShaderSource() {

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D content;

    uniform sampler2D mask;

​

    varying vec2 vTexCoord;

​

    void main(void) {

      gl_FragColor = vec4(texture2D(content, vTexCoord).xyz, 1.) * smoothstep(0.3, 0.9, texture2D(mask, vTexCoord).x);

    }

  `

​

  return [vert, frag]

}

​

function warpShaderSource() {

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D content;

    // uniform sampler2D mask;

    uniform vec2 closestPoint;

​

    varying vec2 vTexCoord;

    float map(float value, float min1, float max1, float min2, float max2) {

      return clamp(

        min2 + (value - min1) * (max2 - min2) / (max1 - min1),

        min(min2, max2),

        max(min2, max2)

      );

    }

    float random(vec2 p) {

      vec3 p3  = fract(vec3(p.xyx) * .1031);

      p3 += dot(p3, p3.yzx + 33.33);

      return fract((p3.x + p3.y) * p3.z);

    }

​

    void main(void) {

      /*float maxR = 0.5;

      float closestR = 0.5;

      vec2 closestPoint = vec2(0.);

      float total = 0.;

      float aOff = 0.; //random(vTexCoord * 123.);

      for (int i = 0; i < 250; i++) {

        float r = float(i)/250.;

        float a = r * ${Math.PI} * (200. + aOff * 0.1);

        r *= maxR;

        vec2 off = vec2(cos(a), sin(a)) * r;

        vec2 pt = vTexCoord + off;

        float val = texture2D(mask, pt).r;

        if (val > 0.3) {

          float weight = 1.; // / (r + 10.);

          closestPoint += pt * weight;

          // closestR += r * weight;

          total += weight;

          // closestPoint = closestR == r ? pt : closestPoint;

        }

      }

      closestPoint /= max(1., total);

      if (total == 0.) {

        closestR = maxR;

      } else {

        closestR = length(closestPoint - vTexCoord);

      }*/

      float maxR = 0.8;

      float closestR = length(closestPoint - vTexCoord);

      //gl_FragColor = vec4(vec3(map(closestR, 0., maxR, 1., 0.)), 1.);

      vec2 offset = (closestPoint - vTexCoord) * 0.8 * pow(map(closestR, 0., maxR, 1., 0.), 4.);

      gl_FragColor = texture2D(content, vTexCoord + offset);

    }

  `

​

  return [vert, frag]

}

​

let cam

let bg

let fg1

let fb2

let captureData

let maskData

let prevMaskData

let fboSize

let tapped = false

​

function preload() {

  font = loadFont('https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTeHuS_nVMrMxCp50SjIw2boKoduKmMEVuLyfMZhrib2Bg-4.ttf')

}

​

async function setup() {

  createCanvas(800, 800, WEBGL)

  pixelDensity(1)

  const isMobile = window.navigator.userAgent && /Mobi|Android/i.test(window.navigator.userAgent)

  // cam = createCapture(VIDEO, { flipped: true })

  cam = createCapture(isMobile ? {

    audio: false,

    video: {

      facingMode: {

        exact: "environment"

      }

    },

  } : VIDEO, { flipped: !isMobile })

  cam.hide()

  fboSize = { width: 800, height: 800 }

  bg = createFramebuffer()

  fg1 = createFramebuffer()

  fg2 = createFramebuffer()

  captureData = createGraphics(fboSize.width, fboSize.height)

  maskData = createGraphics(fboSize.width, fboSize.height)

  prevMaskData = createGraphics(fboSize.width, fboSize.height)

  fbo = createFramebuffer(fboSize)

  maskShader = createShader(...maskShaderSource())

  warpShader = createShader(...warpShaderSource())

  segmenter = await bodySegmentation.createSegmenter(model, segmenterConfig)

}

​

let didRun = false

​

function draw() {

  if (!segmenter) return

  if (!running) {

    running = true

    captureData.clear()

    captureData.image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER)

    let mask

    segmenter.segmentPeople(captureData.elt, segmentationConfig).then((res) => {

      mask = res[0].mask.mask

      return res[0].mask.toImageData()

    }).then((img) => {

      prevMaskData.clear()

      prevMaskData.image(maskData, 0, 0)

      maskData.clear()

      maskData.drawingContext.putImageData(img, 0, 0)

      maskData.filter(BLUR, 2)

      maskData.push()

      // maskData.blendMode(ADD)

      // maskData.tint(255, 180)

      // maskData.image(prevMaskData, 0, 0)

      maskData.pop()

      // maskData.filter(ERODE)

      // maskData.filter(BLUR, 2)

      fbo.begin()

      clear()

      shader(maskShader)

      maskShader.setUniform('content', captureData)

      maskShader.setUniform('mask', maskData)

      noStroke()

      plane(fboSize.width, fboSize.height)

      fbo.end()

      imageMode(CENTER);

      mask.dispose()

      running = false

    })

  }

  imageMode(CENTER);

  bg.draw(() => image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER))

  image(bg, 0, 0)

  image(fg1, 0, 0)

  image(fbo, 0, 0)

  if (!tapped) {

    push()

    noStroke()

    fill(0, 100)

    rectMode(CENTER)

    rect(0, 0, 300, 50)

    textFont(font)

    textAlign(CENTER, CENTER)

    fill(255)

    textSize(30)

    text('Tap to stamp', 0, -4)

    pop()

  }

}

​

function mouseClicked() {

  tapped = true;

  [fg1, fg2] = [fg2, fg1];

  maskData.loadPixels()

  const avgPt = createVector(0, 0)

  let total = 0

  for (let i = 0; i < 1000; i++) {

    const x = floor(random(maskData.width))

    const y = floor(random(maskData.height))

    if (maskData.pixels[(maskData.width * y + x) * 4] > 77) {

      avgPt.add(x/maskData.width, y/maskData.height)

      total++

    }

  }

  avgPt.div(total || 1)

  fg1.draw(() => {

    clear()

    imageMode(CENTER)

​

    push()

    shader(warpShader)

    warpShader.setUniform('content', fg2)

    warpShader.setUniform('closestPoint', [avgPt.x, avgPt.y])

    // warpShader.setUniform('mask', maskData)

    noStroke()

    plane(fg1.width, fg1.height)

    pop()

​

    image(fbo, 0, 0)

  })

}

​

function keyPressed() {

  save('trails.png')

}