Another webcam test

const model = bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation;

const segmenterConfig = {

  runtime: 'tfjs',

  solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@1.0.2'

                // or 'base/node_modules/@mediapipe/selfie_segmentation' in npm.

};

const segmentationConfig = { flipHorizontal: false, landscape: true };

​

let font

​

let segmenter

let capture

let running

​

let maskShader

let warpShader

let stampShader

let bgPicker

let picker

let video

let start

​

let fbo

​

OPC.slider({

  name: 'maxR',

  label: 'Warp',

  min: 0.01,

  max: 0.25,

  value: 0.1

})

OPC.button('myButton', 'Save Image')

​

const vert = `

  precision highp float;

  precision highp int;

​

  attribute vec3 aPosition;

  attribute vec2 aTexCoord;

  attribute vec4 aVertexColor;

​

  uniform mat4 uModelViewMatrix;

  uniform mat4 uProjectionMatrix;

​

  varying vec2 vTexCoord;

​

  void main(void) {

    vec4 viewModelPosition = uModelViewMatrix * vec4(aPosition, 1.0);

    gl_Position = uProjectionMatrix * viewModelPosition;  

    vTexCoord = aTexCoord;

  }

`

​

function maskShaderSource() {

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D content;

    uniform sampler2D mask;

​

    varying vec2 vTexCoord;

​

    void main(void) {

      gl_FragColor = vec4(texture2D(content, vTexCoord).xyz, 1.) * smoothstep(0.3, 0.9, texture2D(mask, vTexCoord).x);

    }

  `

​

  return [vert, frag]

}

​

function stampShaderSource() {

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D content;

​

    varying vec2 vTexCoord;

​

    void main(void) {

      vec4 c = texture2D(content, vTexCoord);

      if (c.a > 0.) {

        gl_FragColor = c / c.a;

      } else {

        gl_FragColor = vec4(0.);

      }

    }

  `

​

  return [vert, frag]

}

​

function warpShaderSource() {

  const frag = `

    precision highp float;

    precision highp int;

​

    uniform sampler2D prev;

    uniform sampler2D next;

    uniform bool first;

    uniform float maxR;

​

    varying vec2 vTexCoord;

    float map(float value, float min1, float max1, float min2, float max2) {

      return clamp(

        min2 + (value - min1) * (max2 - min2) / (max1 - min1),

        min(min2, max2),

        max(min2, max2)

      );

    }

    float random(vec2 p) {

      vec3 p3  = fract(vec3(p.xyx) * .1031);

      p3 += dot(p3, p3.yzx + 33.33);

      return fract((p3.x + p3.y) * p3.z);

    }

    float opSmoothUnion( float d1, float d2, float k )

    {

        float h = clamp( 0.5 + 0.5*(d2-d1)/k, 0.0, 1.0 );

        return mix( d2, d1, h ) - k*h*(1.0-h);

    }

    vec2 smin( float a, float b, float k )

    {

        float h = 1.0 - min( abs(a-b)/(6.0*k), 1.0 );

        float w = h*h*h;

        float m = w*0.5;

        float s = w*k; 

        return (a<b) ? vec2(a-s,m) : vec2(b-s,1.0-m);

    }

​

    void main(void) {

      if (first) {

        gl_FragColor = texture2D(next, vTexCoord);

        return;

      }

      // float maxR = 0.2;

      float OPAQUE = 1.;

      bool onPrev = texture2D(prev, vTexCoord).a == OPAQUE;

      bool onNext = texture2D(next, vTexCoord).a == OPAQUE;

      // float aOff = 0.;

      float aOff = random(vTexCoord * 123.456);

      vec2 closestPrevAngle = vec2(0.);

      float prevAngleSamples = 0.;

      for (int i = 0; i < 100; i++) {

        float a = (float(i)/100. + aOff) * ${2 * Math.PI};

        vec2 off = vec2(cos(a), sin(a)) * maxR;

        vec2 pt = vTexCoord + off;

        float val = texture2D(prev, pt).a;

        if (onPrev ? val < OPAQUE : val >= OPAQUE) {

          closestPrevAngle += off;

          prevAngleSamples++;

        }

      }

      if (dot(closestPrevAngle, closestPrevAngle) > 0.) {

        closestPrevAngle = normalize(closestPrevAngle);

      } else {

        closestPrevAngle = vec2(0.);

      }

      float closestPrevR = maxR;

      for (int i = 0; i < 100; i++) {

        float r = maxR * ((float(i) + aOff)/100.);

        vec2 pt = vTexCoord + closestPrevAngle * r;

        float val = texture2D(prev, pt).a;

        if (onPrev ? val < OPAQUE : val >= OPAQUE) {

          closestPrevR = r;

          break;

        }

      }

      closestPrevR *= onPrev ? -1. : 1.;

      vec2 closestPrev = vTexCoord + closestPrevR * closestPrevAngle;

      vec2 closestNextAngle = vec2(0.);

      float nextAngleSamples = 0.;

      for (int i = 0; i < 100; i++) {

        float a = (float(i)/100. + aOff) * ${2 * Math.PI};

        vec2 off = vec2(cos(a), sin(a)) * maxR;

        vec2 pt = vTexCoord + off;

        float val = texture2D(next, pt).a;

        if (onNext ? val < OPAQUE : val >= OPAQUE) {

          closestNextAngle += off;

          nextAngleSamples++;

        }

      }

      if (dot(closestNextAngle, closestNextAngle) > 0.) {

        closestNextAngle = normalize(closestNextAngle);

      } else {

        closestNextAngle = vec2(0.);

      }

      float closestNextR = maxR;

      for (int i = 0; i < 100; i++) {

        float r = maxR * ((float(i) + aOff)/100.);

        vec2 pt = vTexCoord + closestNextAngle * r;

        float val = texture2D(next, pt).a;

        if (onNext ? val < OPAQUE : val >= OPAQUE) {

          closestNextR = r;

          break;

        }

      }

      closestNextR *= onNext ? -1. : 1.;

      vec2 closestNext = vTexCoord + closestNextR * closestNextAngle;

      // float distortion = min(1. - closestNextR/maxR, 1. - closestPrevR/maxR);

      // vec2 distortionAngle = normalize(closestPrev - closestNext);

      // https://iquilezles.org/articles/smin/

      // vec2 smoothMin = smin(closestPrevR, closestNextR, maxR);

      // float smoothR = clamp(smoothMin.x, -maxR, maxR);

      // float blendFactor = smoothMin.y;

      float smoothR = clamp(opSmoothUnion(closestNextR, closestPrevR, maxR), -maxR, maxR);

      // float smoothR = opSmoothUnion(closestNextR, closestPrevR, maxR);

      // float t = 0.5;

      // closestNextR = max(0., closestNextR);

      // closestPrevR = max(0., closestPrevR);

      // vec2 samplePrev = closestPrev + (smoothR) * (abs(closestPrevR) == maxR ? vec2(0.) : normalize(closestPrev - vTexCoord));

      // vec2 sampleNext = closestNext + abs(smoothR - closestNextR) * (abs(closestNextR) == maxR ? vec2(0.) : normalize(closestNext - vTexCoord));

      // float prevMix = abs(smoothR - closestPrevR)/maxR;

      // vec2 samplePrev = mix(vTexCoord, closestPrev, prevMix);

      // float nextMix = abs(smoothR - closestNextR)/maxR;

      // vec2 sampleNext = mix(vTexCoord, closestNext, nextMix);

      // vec2 sampleNext = vTexCoord + smoothstep(0., maxR, abs(closestNext - vTexCoord)) * nextMix;

      // float diff = map(closestNextR - smoothR, 0., maxR, 0., 1.);

      /*vec2 sampleNext = vTexCoord + (closestNextR > 0.

        ? (closestNext - vTexCoord) * map(closestNextR, 0., maxR, 2.5, 1.)

        : (vTexCoord - closestNext) * -map(closestNextR, -maxR, 0., 0., 2.)

      );*/

      // vec2 dirPrev = closestPrevR > 0. ? (closestPrev - vTexCoord) : (closestNext - vTexCoord);

      vec2 dirPrev = closestPrevAngle * (closestPrevR > 0. ? 1. : -1.);

      if (dirPrev.x != 0. && dirPrev.y != 0.) {

        dirPrev = normalize(dirPrev);

      }

      // vec2 samplePrev = vTexCoord + dirPrev * 0.5 * (closestNextR + closestPrevR) * (smoothR < 0. ? 1. : 0.);

      vec2 samplePrev = vTexCoord + dirPrev * maxR * (1. - abs(closestNextR/maxR)) * (1. - abs(closestPrevR/maxR)) * (smoothR < 0. ? 1. : 0.);

      // vec2 dirNext = closestNextR > 0. ? (closestNext - vTexCoord) : (closestPrev - vTexCoord);

      vec2 dirNext = closestNextAngle * (closestNextR > 0. ? 1. : -1.);

      if (dirNext.x != 0. && dirNext.y != 0.) {

        dirNext = normalize(dirNext);

      }

      // vec2 sampleNext = vTexCoord + dirNext * (clamp(closestNextR, 0., maxR) + clamp(closestPrevR, 0., maxR)) * (smoothR < 0. ? 1. : 0.);

      vec2 sampleNext = vTexCoord + dirNext * maxR * (1. - abs(closestNextR/maxR)) * (1. - abs(closestPrevR/maxR)) * (smoothR < 0. ? 1. : 0.);

      // float t = first ? 1. : (prevMix / (prevMix + nextMix + 0.0001));

      vec4 prevColor = texture2D(prev, samplePrev);

      vec4 nextColor = texture2D(next, sampleNext);

      float weightPrev = map(closestPrevR, maxR, 0., 0., 1.) * (map(closestPrevR, -maxR, maxR*0.1, 1., 0.)); // * prevColor.a;

      float weightNext = map(closestNextR, maxR, 0., 0., 1.) * (map(closestNextR, -maxR, maxR*0.1, 1., 0.)); // * nextColor.a;

      float blendFactor = (weightPrev == 0. && weightNext == 0.) ? 0.5 : (weightNext / (weightPrev + weightNext));

      // float blendFactor = 0.;

      // float tPrev = 1. - prevColor.a;

      // float tNext = 1. - nextColor.a;

      // float t = (tPrev) / (tPrev + tNext + 0.0001);

      // float t = 0.5;

      gl_FragColor = mix(prevColor, nextColor, clamp(blendFactor, 0., 1.)) * (smoothR < 0. ? 1. : 0.);

      // gl_FragColor = vec4(vec3(length(sampleNext - vTexCoord)), 1.);

      // gl_FragColor = texture2D(next, vTexCoord);

      // gl_FragColor = (prevColor + nextColor);

      // if (prevColor.a + nextColor.a > 0.) gl_FragColor /= prevColor.a + nextColor.a;

      // gl_FragColor = (prevColor * (1. - nextColor.a) + nextColor) * (smoothR <= 0. ? 1. : 0.);

      // gl_FragColor = nextColor * (smoothR <= 0. ? 1. : 0.);

      // gl_FragColor = vec4(1.) * clamp(abs(smoothR - closestNextR)/maxR, 0., 1.);

      // gl_FragColor = texture2D(next, closestNext);

    }

  `

​

  return [vert, frag]

}

​

let cam

let bg

let fg1

let fb2

let captureData

let maskData

let prevMaskData

let fboSize

let tapped = false

​

function preload() {

  font = loadFont('https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTeHuS_nVMrMxCp50SjIw2boKoduKmMEVuLyfMZhrib2Bg-4.ttf')

}

​

let doStamp = false

async function setup() {

  createCanvas(800, 800, WEBGL)

  pixelDensity(1)

  const isMobile = window.navigator.userAgent && /Mobi|Android/i.test(window.navigator.userAgent)

  // cam = createCapture(VIDEO, { flipped: true })

  cam = createCapture(isMobile ? {

    audio: false,

    video: {

      facingMode: {

        exact: "environment"

      }

    },

  } : VIDEO, { flipped: !isMobile })

  cam.hide()

  fboSize = { width: 800, height: 800 }

  bg = createFramebuffer()

  fg1 = createFramebuffer()

  fg2 = createFramebuffer()

  captureData = createGraphics(fboSize.width, fboSize.height)

  maskData = createGraphics(fboSize.width, fboSize.height)

  prevMaskData = createGraphics(fboSize.width, fboSize.height)

  fbo = createFramebuffer(fboSize)

  maskShader = createShader(...maskShaderSource())

  warpShader = createShader(...warpShaderSource())

  stampShader = createShader(...stampShaderSource())

  segmenter = await bodySegmentation.createSegmenter(model, segmenterConfig)

}

​

let didRun = false

​

function draw() {

  if (!segmenter) return

  if (doStamp && !running) {

    // [fg1, fg2] = [fg2, fg1];

    fg2.draw(() => {

      clear()

      shader(stampShader)

      stampShader.setUniform('content', fg1)

      noStroke()

      plane(fg1.width, fg1.height)

    })

    doStamp = false

  }

  if (!running) {

    running = true

    captureData.clear()

    captureData.image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER)

    let mask

    segmenter.segmentPeople(captureData.elt, segmentationConfig).then((res) => {

      mask = res[0].mask.mask

      const data = res[0].mask.toImageData()

      mask.dispose()

      return data

    }).then((img) => {

      prevMaskData.clear()

      prevMaskData.image(maskData, 0, 0)

      maskData.clear()

      maskData.drawingContext.putImageData(img, 0, 0)

      maskData.filter(BLUR, 2)

      maskData.push()

      // maskData.blendMode(ADD)

      // maskData.tint(255, 180)

      // maskData.image(prevMaskData, 0, 0)

      maskData.pop()

      // maskData.filter(ERODE)

      // maskData.filter(BLUR, 2)

      fbo.begin()

      clear()

      shader(maskShader)

      maskShader.setUniform('content', captureData)

      maskShader.setUniform('mask', maskData)

      noStroke()

      plane(fboSize.width, fboSize.height)

      fbo.end()

      imageMode(CENTER);

​

      requestAnimationFrame(() => {

        fg1.draw(() => {

          clear()

          imageMode(CENTER)

​

          push()

          shader(warpShader)

          warpShader.setUniform('prev', fg2)

          warpShader.setUniform('next', fbo)

          warpShader.setUniform('first', !tapped)

          warpShader.setUniform('maxR', maxR)

          noStroke()

          plane(fg1.width, fg1.height)

          pop()

​

          // image(fg2, 0, 0)

          // image(fbo, 0, 0)

        })

​

        running = false

      })

    })

  }

  imageMode(CENTER);

  clear()

  bg.draw(() => image(cam, 0, 0, fboSize.width, fboSize.height, 0, 0, cam.width, cam.height, COVER))

  image(bg, 0, 0)

  image(fg1, 0, 0)

  // image(fg2, 0, 0)

  // image(fbo, 0, 0)

  if (!tapped) {

    push()

    noStroke()

    fill(0, 100)

    rectMode(CENTER)

    rect(0, 0, 300, 50)

    textFont(font)

    textAlign(CENTER, CENTER)

    fill(255)

    textSize(30)

    text('Tap to stamp', 0, -4)

    pop()

  }

}

​

function mouseClicked() {

  tapped = true;

  doStamp = true;

}

​

function buttonReleased() {

  save('trails.png')

}