Check if window size is different than rendering window size, it can be a problem with the window title bar.
Then what are the mouse coords? If you use a standard perspective matrix why wont you use some simple math to involve ray creation. From eye pos to near plane
Like in this
inline vec3 GetDirectionFromScreen(mat4 modelview, TSpecCamera * FPP_CAM, int x, int y, int sw, int sh, float fov, float z_near, float aspect)//, float z_far)
{
mat4 mvm = modelview;
mvm.Transpose();
vec3 dirX, dirY;
dirX.x = mvm.m[0];
dirX.y = mvm.m[4];
dirX.z = mvm.m[8];
dirY.x = mvm.m[1];
dirY.y = mvm.m[5];
dirY.z = mvm.m[9];
// float aspect = float (SCREEN_WIDTH) / float (SCREEN_HEIGHT);
float a = fov / 2.0;
float cotangent = 1.0 / tan( a * imopi );
float ax = z_near / cotangent;
float screen_w = 2.0*ax;
float screen_h = screen_w;// * yratio;
screen_w = screen_w * aspect;
float scr_coord_x = float(x) / float(sw);
float scr_coord_y = float(sh - y) / float(sh);
vec3 dir = FPP_CAM->ReturnFrontVector();
//move to lower left corner
vec3 start_pos = (FPP_CAM->pos + dir * z_near) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));
vec3 start = start_pos + (dirX * (screen_w * scr_coord_x)) + (dirY * (screen_h * scr_coord_y));
return Normalize( vectorAB(FPP_CAM->pos, start) );
}
Anyway its dor openglbased subsystem
Anyway there may be isaues with 1.0- mouse.y and fov is not fovy byt horizontal fov
