Changing the xoff coordinates of the mesh after changing the angle to pan in correct direction

Question

I am struggling with setting up coordinates so that when you change the angle, the mesh moves like a mouse. So if I move the mouse up, the mesh also goes up and not at the angle I changed. I will be grateful for any hints. Thanks. This is my code:

let angle = 0.0;
let dim = 10
let sz;
let xoff;
let yoff;
function setup() {
  createCanvas(400, 400);
  sz = width/ dim;
}

function mouseDragged() {
  xoff += (mouseX - pmouseX)/10;
  yoff += (mouseY - pmouseY)/10;
}

function draw() {
  background(255);
  
  for (let i = 0; i < dim+2; i++) {
    for (let j = 0; j < dim+2; j++) {

      let x = ((pmouseX + j * sz) % (width+sz)) - sz;
      if (x < -sz) x += width+sz;

      let y = ((pmouseY + i * sz) % (height+sz)) - sz;
      if (y < -sz) y += height+sz;
push();
imageMode(CENTER);
rectMode(CENTER);
translate( width / 2  , height / 2  );
rotate(angle);
translate( -width / 2 , -height / 2  );
      rect(x, y, sz, sz);
      text(i * 10 + j, x + sz/2, y + sz/2);
      pop(); 
    }
  }
}