Genuary 31: Camera Depth - OpenProcessing

mySketch.js
style.css
index.html
xxxxxxxxxx
 
let capture
let estimator
let config
let task
let depthCnv
let depthShader
let fbo
let camFbo
let depthFbo
​
async function setup() {
  createCanvas(windowWidth, windowHeight, WEBGL)
  // setAttributes({ textureFiltering: NEAREST })
  fbo = createFramebuffer()
  capture = createCapture({
    video: {
      mandatory: {
        minWidth: 1280,
        minHeight: 720
      },
      // optional: [{ maxFrameRate: 10 }]
    }
  });
  capture.size(1280, 720);
  capture.hide();
  // console.log(capture.width, capture.height)
  depthFbo = createFramebuffer({ width: capture.width, height: capture.height })
  camFbo = createFramebuffer({ width: capture.width, height: capture.height })
  const model = depthEstimation.SupportedModels.ARPortraitDepth;
  estimator = await depthEstimation.createEstimator(model);
  config = { minDepth: 0.1, maxDepth: 0.99 }
  
  depthShader = baseMaterialShader().modify({
    uniforms: {
      'sampler2D depthMap': () => depthFbo,
      'vec2 size': () => [camFbo.width, camFbo.height],
    },
    'Inputs getPixelInputs': `(Inputs inputs) {
      float depth = texture(depthMap, inputs.texCoord).r;
      
      for (int i = 0; i < 12; i++) {
        float a = float(i)/12. * ${TWO_PI};
        vec2 coord = inputs.texCoord + 8.*vec2(cos(a), sin(a))/size;
        if (texture(depthMap, coord).r < 0.1) {
          depth = 1.;
          break;
        }
      }
      
      gl_FragDepth = depth;
      return inputs;
    }`
  })
}
​
function mouseClicked() {
  fbo.draw(() => clearDepth())
}
​
function draw() {
  if (!task && estimator) {
    if (!capture.elt.width || !capture.elt.height) return
    camFbo.draw(() => {
      imageMode(CENTER)
      image(capture, 0, 0, camFbo.width, camFbo.height)
    })
    task = estimator.estimateDepth(capture.elt, config)
      .then((depthMap) => depthMap.toCanvasImageSource())
      .then((img) => {
        depthCnv = new p5.Element(img)
        depthCnv.width = img.width
        depthCnv.height = img.height
      
        depthFbo.begin()
        background(0)
        imageMode(CENTER)
        image(depthCnv, 0, 0, depthFbo.width, depthFbo.height)
        depthFbo.end()
      
        fbo.begin()
        noStroke()
        texture(camFbo)
        shader(depthShader)
        scale(max(width/camFbo.width, height/camFbo.height))
        plane(camFbo.width, camFbo.height)
        fbo.end()
      
        push()
        imageMode(CENTER)
        scale(-1, 1)
        image(fbo, 0, 0)
        pop()
        task = undefined
      })
  }
}