How to calculate lookAt matrix for 3D rotation with mouseMoveEvent?

Question

I have been trying to calculate view matrix with lookAt function for each mouseMoveEvent. Right now what I can do is rotate the image but the rotation is not proper when I zoom out.I change the value of FOV in projection matrix for each wheelEvent.

Here is my vertex shader code.

    const char *vsrc = "attribute vec3 vertexIn; \
    attribute vec3 textureIn; \
    varying vec3 textureOut;  \
    uniform mat4 model;       \
    uniform mat4 view;       \
    uniform mat4 projection;  \
    void main(void)           \
    {                         \
        gl_Position = projection*view*model*vec4(vertexIn/1000.0,1.0); \
        textureOut = textureIn; \
    }";

Here is my fragment shader code.

    const char *fsrc = "#ifdef GL_ES\n"
                 "precision highp float;\n"
                 "#endif\n"

                 "varying vec3 textureOut;\n"

                 "void main()\n"
                 "{\n"
                   "   gl_FragColor = vec4(textureOut.bgr,1.0);"
                   "}";

The function which is called for each Event is

void ABC::wheelEvent(QWheelEvent *event)
{
    if(event->delta()>0)
    {
        FoV -= 0.5f;                         //for zoom out have to decrease FoV
    }
    else if(event->delta()<=0)
    {
        FoV += 0.5f;                         //for zoom in have to increase FoV
    }

    if (FoV < 0.5f)              //maximum zoom
         FoV = 0.5f;
    else if(FoV > 177.5)         //minimum zoom
        FoV = 177.5;
    ProjectionMatrix.setToIdentity();
    ProjectionMatrix.perspective(FoV, 4.0f / 3.0f, 0.1f, 100.0f);
}

void ABC::mousePressEvent(QMouseEvent *event)
{
    lastMousePos = event->pos();
}

void ABC::mouseMoveEvent(QMouseEvent *event){

    // Compute new orientation
    horizontalAngle += mouseSpeed * float(event->pos().x() - lastMousePos.x() );
    verticalAngle   += mouseSpeed * float(event->pos().y() - lastMousePos.y() );
     lastMousePos = event->pos();

    // Direction : Spherical coordinates to Cartesian coordinates conversion
    direction = QVector3D(                                                               //to find out the direction for camera to look at
        cos(verticalAngle) * sin(horizontalAngle),
        sin(verticalAngle),
        cos(verticalAngle) * cos(horizontalAngle)
    );

    ViewMatrix.setToIdentity();
    ViewMatrix.lookAt(
                                direction,           // Camera is here
                                QVector3D(0.0,0.0,0.0),             // and looks here
                                QVector3D(0.0,1.0,0.0)                  // Head is up (set to 0,-1,0 to look upside-down)
                           );
}

I'm new to 3D rendering and forgive me if I did some lame mistake. Can anyone please tell me what I'm doing wrong here or provide me with some examples so I can rectify.

The link to the video: https://www.kapwing.com/videos/6274ae966e15330062fb7fba