output: fix transform matrix for 90/270 rotations

We need to adjust the second translation depending on the transform
we applied.

Fixes: 9601a2abf0 ("output: improve transform matrix calculation"
Closes: https://github.com/swaywm/wlroots/issues/2774

backend/drm/drm.c

@@ -938,11 +938,17 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		float output_matrix[9];
 		wlr_matrix_identity(output_matrix);
 		if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			struct wlr_box tr_size = {
+				.width = plane->surf.width,
+				.height = plane->surf.height,
+			};
+			wlr_box_transform(&tr_size, &tr_size, output->transform, 0, 0);
+
 			wlr_matrix_translate(output_matrix, plane->surf.width / 2.0,
 					plane->surf.height / 2.0);
 			wlr_matrix_transform(output_matrix, output->transform);
-			wlr_matrix_translate(output_matrix, - plane->surf.width / 2.0,
+			wlr_matrix_translate(output_matrix, - tr_size.width / 2.0,
-					- plane->surf.height / 2.0);
+					- tr_size.height / 2.0);
 		}
 
 		float matrix[9];

backend/wayland/output.c

@@ -434,9 +434,13 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 		float output_matrix[9];
 		wlr_matrix_identity(output_matrix);
 		if (wlr_output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			struct wlr_box tr_size = { .width = width, .height = height };
+			wlr_box_transform(&tr_size, &tr_size, wlr_output->transform, 0, 0);
+
 			wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
 			wlr_matrix_transform(output_matrix, wlr_output->transform);
-			wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+			wlr_matrix_translate(output_matrix,
+				- tr_size.width / 2.0, - tr_size.height / 2.0);
 		}
 
 		float matrix[9];

backend/x11/output.c

@@ -418,9 +418,13 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 	float output_matrix[9];
 	wlr_matrix_identity(output_matrix);
 	if (output->wlr_output.transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		struct wlr_box tr_size = { .width = width, .height = height };
+		wlr_box_transform(&tr_size, &tr_size, output->wlr_output.transform, 0, 0);
+
 		wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
 		wlr_matrix_transform(output_matrix, output->wlr_output.transform);
-		wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+		wlr_matrix_translate(output_matrix,
+			- tr_size.width / 2.0, - tr_size.height / 2.0);
 	}
 
 	float matrix[9];

types/wlr_output.c

@@ -141,14 +141,16 @@ void wlr_output_update_enabled(struct wlr_output *output, bool enabled) {
 }
 
 static void output_update_matrix(struct wlr_output *output) {
-	float width = output->width / 2.f;
-	float height = output->height / 2.f;
-
 	wlr_matrix_identity(output->transform_matrix);
 	if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
-		wlr_matrix_translate(output->transform_matrix, width, height);
+		int tr_width, tr_height;
+		wlr_output_transformed_resolution(output, &tr_width, &tr_height);
+
+		wlr_matrix_translate(output->transform_matrix,
+			output->width / 2.0, output->height / 2.0);
 		wlr_matrix_transform(output->transform_matrix, output->transform);
-		wlr_matrix_translate(output->transform_matrix, -width, -height);
+		wlr_matrix_translate(output->transform_matrix,
+			- tr_width / 2.0, - tr_height / 2.0);
 	}
 }