Compute transformation matrix based on two orientations

Question

I have two Orientation3D instances.

public class Orientation3D
{
    public Orientation3D(Point point, Vector directionVector, Vector upVector)
        : base(point, directionVector)
    {
        Position = point;
        Direction = directionVector;
        UpVector = upVector;
    }

    public Vector UpVector { get; private set; }
    
    public Point Position { get; private set; }

    public Vector Direction { get; private set; }
}

And now I have to calculate a homogenous affine transformation matrix (that name is all that is left from my days at the university quite some years ago...) that would transform one orienation to another one.

Use this as an example:

    [Test]
    public void Should_calculate_transformation()
    {
        var orientationA = new Orientation3D(new Point(1, 2, 3), new Vector(1, 0, 0), new Vector(0, 0, 1));
        var orientationB = new Orientation3D(new Point(4, 5, 6), new Vector(0, 1, 0), new Vector(1, 0, 0));

        var transformation = CalculateTransformation(orientationA, orientationB);
    }

    private Matrix4x4 CalculateTransformation(
        Orientation3D orientationA,
        Orientation3D orientationB)
    {
        //// I don't know what to do here....
    }

Could you point me to the right direction? Or describe the calculation that is necessary? Any help is appreciated.

Note: Usually a colleague of mine is responsible for these calculations, but unfortunately he is on paternity leave.

I can show some methods he already wrote. But as you see, these methods are only working for a special case (look at the assumptions on position, target and upVector), and I do not know, how to modify them for my case.

public static class PositionDataToMatrixConverter
{
    /// <summary>
    /// Converts the orientation into a transformation
    /// Assumptions are : Default-Position is 0,0,0, 
    ///                   Default-forward-vector is 0,0,1
    ///                   Default-UpVector is 0,1,0
    /// </summary>
    public static Matrix4x4 Convert(Point postion, Vector forward, Vector up)
    {
        var target = postion + (forward * -1);

        var mat = Math3D.CalculateLookAtMartix(postion, target, up);
        mat.Invert();
        return mat;
    }

    /// <summary>
    /// Converts the orientation into a transformation
    /// Assumptions are : Default-Position is 0,0,0, 
    ///                   Default-forward-vector is 0,0,1
    ///                   Default-UpVector is 0,1,0
    /// </summary>
    public static Matrix4x4 Convert(Orientation3D positionInfo)
    {
        return Convert(positionInfo.Position, positionInfo.Direction, positionInfo.UpVector);
    }
}

public static class Math3D
{
    public static Matrix4x4 CalculateLookAtMartix(Point position, Point target, Vector upVector)
    {
        // https://www.opengl.org/sdk/docs/man2/xhtml/gluLookAt.xml
        var retval = new Matrix4x4();

        retval.SetIdentity();
        var viewDirection = Point.Subtract(target, position);
        viewDirection.Normalize();
        upVector.Normalize();
        var viewXaxis = Vector.CrossProduct(viewDirection, upVector);
        viewXaxis.Normalize();
        var sNorm = new Vector(viewXaxis);
        sNorm.Normalize();
        var viewYaxis = Vector.CrossProduct(sNorm, viewDirection);
        retval.Set(0, 0, viewXaxis.X);
        retval.Set(1, 0, viewXaxis.Y);
        retval.Set(2, 0, viewXaxis.Z);
        retval.Set(0, 1, viewYaxis.X);
        retval.Set(1, 1, viewYaxis.Y);
        retval.Set(2, 1, viewYaxis.Z);
        retval.Set(0, 2, -viewDirection.X);
        retval.Set(1, 2, -viewDirection.Y);
        retval.Set(2, 2, -viewDirection.Z);
        var transMat = new Matrix4x4();
        transMat.SetIdentity();
        transMat.Translate(Point.Subtract(new Point(0, 0, 0), position));
        return Matrix4x4.Multiply(transMat, retval);
    }
}