CustomPixel+AI

// "Custom Picture Element", with AI augmentation.

// This project ingests 3 images: rgb, segment, depth.

// All three images are required to have the same dimensions.

// 1. rgb.png: An rgb photograph of an elephant

//    From: https://www.britannica.com/animal/elephant-mammal

// 2. depth.png: a depth image, from HKU's "Depth Anything V2".

// 3. segment.png: a semantic segmentation mask image, created  

//    with the "Segment Anything 2 Model" (SAM) by Meta AI.

// For full details, see:

// https://github.com/golanlevin/60-212/blob/main/lectures/comfy/image_analysis/readme.md

​

let imgRgb, imgDepth, imgSegment;

let gridSize = 24;

​

// Preload the three images

function preload(){

  imgRgb = loadImage("rgb.png"); 

  imgDepth = loadImage("depth.png"); 

  imgSegment = loadImage("segment.png"); 

}

​

function setup() {

  // This is also the same as the images' sizes.

  createCanvas(964, 640); 

}

​

function draw() {

  background(0,0,40);

  const nRows = int(imgRgb.height / gridSize);

  const nCols = int(imgRgb.width / gridSize); 

  for (let row=0; row<nRows; row++){

    for (let col=0; col<nCols; col++){

      let px = (col + 0.5) * gridSize; 

      let py = (row + 0.5) * gridSize;

      drawCustomPictureElement (px,py);

    }

  }

}

​

// Use all 5 channels of image data: RGB + Depth + Segmentation

// to do something interesting. In the design shown here, e.g,: 

// * Elements inside our segmentation mask are colored differently. 

// * Elements that are 'further away' (from depth) appear smaller. 

function drawCustomPictureElement (px,py){

  let colorOfRgbPixel = imgRgb.get(px, py); 

  let colorOfDepthPixel = imgDepth.get(px, py); 

  let colorOfSegmentPixel = imgSegment.get(px, py); 

  let bIsElephant = (colorOfSegmentPixel[0] > 0);

  fill (bIsElephant ? 'HotPink' : colorOfRgbPixel); 

  let r = red(colorOfDepthPixel); // depth is a grayscale image

  let squareSize = map(r,0,255, 0.2,1.0) * gridSize; 

  noStroke();

  rectMode(CENTER); 

  square(px,py, squareSize); 

}