output: improve transform matrix calculation

Compute only the transform matrix in the output. The projection matrix
will be calculated inside the gles2 renderer when we start rendering.

The goal is to help the pixman rendering process.

backend/drm/drm.c

@@ -895,10 +895,6 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		}
 	}
 
-	float hotspot_proj[9];
-	wlr_matrix_projection(hotspot_proj, plane->surf.width,
-		plane->surf.height, output->transform);
-
 	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
 	wlr_box_transform(&hotspot, &hotspot,
 		wlr_output_transform_invert(output->transform),
@@ -939,8 +935,19 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		struct wlr_box cursor_box = { .width = width, .height = height };
 
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, plane->surf.width / 2.0,
+					plane->surf.height / 2.0);
+			wlr_matrix_transform(output_matrix, output->transform);
+			wlr_matrix_translate(output_matrix, - plane->surf.width / 2.0,
+					- plane->surf.height / 2.0);
+		}
+
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, hotspot_proj);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0,
+				output_matrix);
 
 		wlr_renderer_begin(rend, plane->surf.width, plane->surf.height);
 		wlr_renderer_clear(rend, (float[]){ 0.0, 0.0, 0.0, 0.0 });

backend/drm/renderer.c

@@ -156,7 +156,7 @@ static struct wlr_buffer *drm_surface_blit(struct wlr_drm_surface *surf,
 	}
 
 	float mat[9];
-	wlr_matrix_projection(mat, 1, 1, WL_OUTPUT_TRANSFORM_NORMAL);
+	wlr_matrix_identity(mat);
 
 	wlr_renderer_begin(renderer, surf->width, surf->height);
 	wlr_renderer_clear(renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });

backend/wayland/output.c

@@ -431,11 +431,16 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 			.height = height,
 		};
 
-		float projection[9];
-		wlr_matrix_projection(projection, width, height, wlr_output->transform);
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (wlr_output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+			wlr_matrix_transform(output_matrix, wlr_output->transform);
+			wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+		}
 
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 		wlr_renderer_begin(backend->renderer, width, height);
 		wlr_renderer_clear(backend->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });

backend/x11/output.c

@@ -415,11 +415,16 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 		.height = height,
 	};
 
-	float projection[9];
-	wlr_matrix_projection(projection, width, height, output->wlr_output.transform);
+	float output_matrix[9];
+	wlr_matrix_identity(output_matrix);
+	if (output->wlr_output.transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+		wlr_matrix_transform(output_matrix, output->wlr_output.transform);
+		wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+	}
 
 	float matrix[9];
-	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 	wlr_renderer_begin(x11->renderer, width, height);
 	wlr_renderer_clear(x11->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });

include/render/gles2.h

@@ -34,6 +34,7 @@ struct wlr_gles2_tex_shader {
 struct wlr_gles2_renderer {
 	struct wlr_renderer wlr_renderer;
 
+	float projection[9];
 	struct wlr_egl *egl;
 	int drm_fd;
 

render/gles2/renderer.c

@@ -191,6 +191,10 @@ static void gles2_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 	renderer->viewport_width = width;
 	renderer->viewport_height = height;
 
+	// refresh projection matrix
+	wlr_matrix_projection(renderer->projection, width, height,
+			WL_OUTPUT_TRANSFORM_NORMAL);
+
 	// enable transparency
 	glEnable(GL_BLEND);
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
@@ -271,7 +275,8 @@ static bool gles2_render_subtexture_with_matrix(
 	}
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -325,7 +330,8 @@ static void gles2_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 		gles2_get_renderer_in_context(wlr_renderer);
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -355,7 +361,8 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 		gles2_get_renderer_in_context(wlr_renderer);
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -579,7 +586,7 @@ static bool gles2_blit_dmabuf(struct wlr_renderer *wlr_renderer,
 
 	// TODO: use ANGLE_framebuffer_blit if available
 	float mat[9];
-	wlr_matrix_projection(mat, 1, 1, WL_OUTPUT_TRANSFORM_NORMAL);
+	wlr_matrix_identity(mat);
 
 	wlr_renderer_begin(wlr_renderer, dst_attr->width, dst_attr->height);
 	wlr_renderer_clear(wlr_renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });

types/wlr_output.c

@@ -141,8 +141,15 @@ void wlr_output_update_enabled(struct wlr_output *output, bool enabled) {
 }
 
 static void output_update_matrix(struct wlr_output *output) {
-	wlr_matrix_projection(output->transform_matrix, output->width,
-		output->height, output->transform);
+	float width = output->width / 2.f;
+	float height = output->height / 2.f;
+
+	wlr_matrix_identity(output->transform_matrix);
+	if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		wlr_matrix_translate(output->transform_matrix, width, height);
+		wlr_matrix_transform(output->transform_matrix, output->transform);
+		wlr_matrix_translate(output->transform_matrix, -width, -height);
+	}
 }
 
 void wlr_output_enable(struct wlr_output *output, bool enable) {