Body segmentation filter using ml5

Question

I'm trying to make a webcam filter which would do body segmentation to mask the background behind a person and place a picture in the background. Something like backgrounds option on Zoom or other platforms. I was thinking of using ml5 and BodyPix but I only reached this point (https://github.com/ml5js/ml5-library/blob/main/examples/p5js/BodyPix/BodyPix_Webcam/sketch.js):

let bodypix;
let video;
let segmentation;

const options = {
  outputStride: 8, // 8, 16, or 32, default is 16
  segmentationThreshold: 0.3, // 0 - 1, defaults to 0.5
};

function preload() {
  bodypix = ml5.bodyPix(options);
}

function setup() {
  createCanvas(320, 240);
  // load up your video
  video = createCapture(VIDEO, videoReady);
  video.size(width, height);
  
}

function videoReady() {
  bodypix.segment(video, gotResults);
}

function draw() {
  background(0);
  if (segmentation) {
    image(segmentation.backgroundMask, 0, 0, width, height);
  }
}

function gotResults(error, result) {
  if (error) {
    console.log(error);
    return;
  }
  segmentation = result;
  bodypix.segment(video, gotResults);
}

Anyone has any solutions for placing an image in the background? Maybe there's another way to code such thing? Thanks!

Answer 1

You are very close. The object returned from the segment function gives you two PImages that are inverse masks of each other for just this purpose. The .backgroundMask masks out the background and all non recognized bodies have pixels with 0 alpha. The .personMask does the opposite and only the background shows. See the reference here: https://github.com/ml5js/ml5-library/blob/main/docs/reference/bodypix.md

let bodypix;
let video;
let segmentation;
let backgroundImage; // start with the variable

const options = {
  outputStride: 8, 
  segmentationThreshold: 0.3,
};

function preload() {
  bodypix = ml5.bodyPix(options);
  backgroundImage = loadImage('YOURFILENAME'); // load your file here
}

function setup() {
  createCanvas(320, 240);
  video = createCapture(VIDEO, videoReady);
  video.size(width, height);
}

function videoReady() {
  bodypix.segment(video, gotResults);
}

function draw() {
  background(0);
  image(backgroundImage,0,0,width,height); // this is the background drawn before the subject is drawn
  if (segmentation) {
    image(segmentation.backgroundMask, 0, 0, width, height); 
  }
}

function gotResults(error, result) {
  if (error) {
    console.log(error);
    return;
  }
  segmentation = result;
  bodypix.segment(video, gotResults);
}

Here is a working example: https://editor.p5js.org/[email protected]/sketches/af83XcLXZ