Loading video frames into javascript in google colab

Question

I wish to present a video, frame by frame with the possibility of flipping between frames in google colab. I also wish to mark some rectangles and get the x,y,w,h of each rectangle.

As far as I could understand, the only way to click the output and gather data in google Collab is via a javascript call, which I am new to. I have split my video into a list called frames and written the following python\javascript code for my task:

def get_fxy_val(base64imgs):
  js = Javascript('''  
  async function showImage(base64) {
        var canvas = document.createElement('canvas');
        var rect = {};
        var drag = false;
        var finish = false;
        var clicks = [];
        var img = document.createElement('img');                
        var frame = 0
        var ctx = canvas.getContext('2d');
        document.body.appendChild(canvas);
        img = new Image();
        img.onload = function () { 
            canvas.height = img.height;
            canvas.width = img.width;
            ctx.drawImage(img, 0, 0); 
        };
        img.src = 'data:image/jpeg;base64,'+base64[frame];

        var prev = document.createElement('button');
        prev.innerHTML = '<';
        prev.onclick = function() {
            frame = frame -1;
            if(frame<0){
                frame = 0;
            }
            else{
                // console.log(frame)
                img.src = 'data:image/jpeg;base64,'+base64[frame];
            };
        };
        document.body.appendChild(prev);
        var next = document.createElement('button');
        next.innerHTML = '>';
        next.onclick = function() {
            frame = frame+1;
            if (frame>base64.length-1) {
                // console.log(frame)
                frame = base64.length-1
            }
            else{
                img.src = 'data:image/jpeg;base64,'+base64[frame];
            }
        };
        document.body.appendChild(next);
        canvas.addEventListener('mousedown', mouseDown, false);
        canvas.addEventListener('mouseup', mouseUp, false);
        canvas.addEventListener('mousemove', mouseMove, false);

        function mouseDown(e) {
            rect.startX = e.pageX - this.offsetLeft;
            rect.startY = e.pageY - this.offsetTop;
            drag = true;
        }

        function mouseUp() { 
            drag = false;
            ctx.clearRect(0, 0, canvas.width, canvas.height);
            ctx.drawImage(img, 0, 0);
            if (rect.w<0){
                rect.startX = rect.startX + rect.w
                rect.w = rect.w*(-1)
            }
            if (rect.h<0){
                rect.startY = rect.startY + rect.h
                rect.h = rect.h*(-1)
            }
            ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
        }

        function mouseMove(e) {
            if (drag) {
                rect.w = (e.pageX - this.offsetLeft) - rect.startX;
                rect.h = (e.pageY - this.offsetTop) - rect.startY;
                ctx.strokeStyle = 'red';
                ctx.lineWidth = 2;
                ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
            }
        }

        var saver = document.createElement('button');
        saver.innerHTML = 'save rectangle';
        saver.onclick = function() {
            clicks.push([frame, rect.startX, rect.startY, rect.w, rect.h]);
            ctx.clearRect(0, 0, canvas.width, canvas.height);
            ctx.drawImage(img, 0, 0);
        };
        document.body.appendChild(saver);
        var finisher = document.createElement('button');
        finisher.innerHTML = 'Done!';
        finisher.onclick = function() {
        };
        document.body.appendChild(finisher);

        await new Promise((resolve) => finisher.onclick = resolve);
        document.body.removeChild(finisher)
        document.body.removeChild(saver)
        document.body.removeChild(next)
        document.body.removeChild(prev)
        document.body.removeChild(canvas)
        return clicks 
    }
  ''')
  display(js)
  imgs = ["'"+b64img+"'" for b64img in base64imgs]
  imgs = '[' + ','.join(imgs) +']'  
  data = eval_js(f"showImage({imgs})")
  return data  

I call it using:

base64imgs = [base64.b64encode(cv2.imencode('.jpg', img)[1]).decode() for img in frames]
data = get_fxy_val(base64imgs)

Everything works well for a small number of images, however for the full video Collab crushes and I cannot manage to find any logs to verify what is going on.

Two thoughts I had, which I do not know how to implement:

1. Reduce the images' memory size somehow. I cannot harm the aspect ratio so downsampling is not a possibility.
2. Load images directly from memory inside the javascript code. I have no idea how to implement this or if it is even possible.

Can I implement any of these? Is there another approach?

B.T.W, my video has images with size h,w,c = (1080, 1920, 3).

Answer 1

try:

=INDEX(QUERY(IFERROR(SPLIT(FLATTEN(A2:A&"×"&B2:B&"×"&
 REGEXEXTRACT(C1:E1, "^.+? ")&"×"&C2:E), "×")), "where Col4 is not null"))