backend/drm: fix hw cursor position on rotated and scaled outputs

output: add wlr_output_transformed_resolution

backend/drm/drm.c

@@ -591,11 +591,10 @@ static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 	};
 	enum wl_output_transform transform =
 		wlr_output_transform_invert(output->transform);
-	struct wlr_box transformed_hotspot;
 	wlr_box_transform(&hotspot, transform,
-		plane->surf.width, plane->surf.height, &transformed_hotspot);
-	plane->cursor_hotspot_x = transformed_hotspot.x;
-	plane->cursor_hotspot_y = transformed_hotspot.y;
+		plane->surf.width, plane->surf.height, &hotspot);
+	plane->cursor_hotspot_x = hotspot.x;
+	plane->cursor_hotspot_y = hotspot.y;
 
 	if (!update_pixels) {
 		// Only update the cursor hotspot
@@ -660,20 +659,18 @@ static bool wlr_drm_connector_move_cursor(struct wlr_output *output,
 	struct wlr_box box = { .x = x, .y = y };
 
 	int width, height;
-	wlr_output_effective_resolution(output, &width, &height);
+	wlr_output_transformed_resolution(output, &width, &height);
 
 	enum wl_output_transform transform =
 		wlr_output_transform_invert(output->transform);
-	struct wlr_box transformed_box;
-	wlr_box_transform(&box, transform, width, height, &transformed_box);
+	wlr_box_transform(&box, transform, width, height, &box);
 
 	if (plane != NULL) {
-		transformed_box.x -= plane->cursor_hotspot_x;
-		transformed_box.y -= plane->cursor_hotspot_y;
+		box.x -= plane->cursor_hotspot_x;
+		box.y -= plane->cursor_hotspot_y;
 	}
 
-	bool ok = drm->iface->crtc_move_cursor(drm, conn->crtc, transformed_box.x,
-		transformed_box.y);
+	bool ok = drm->iface->crtc_move_cursor(drm, conn->crtc, box.x, box.y);
 	if (ok) {
 		wlr_output_update_needs_swap(output);
 	}

include/wlr/types/wlr_output.h

@@ -106,6 +106,14 @@ void wlr_output_set_transform(struct wlr_output *output,
 void wlr_output_set_position(struct wlr_output *output, int32_t lx, int32_t ly);
 void wlr_output_set_scale(struct wlr_output *output, float scale);
 void wlr_output_destroy(struct wlr_output *output);
+/**
+ * Computes the transformed output resolution.
+ */
+void wlr_output_transformed_resolution(struct wlr_output *output,
+	int *width, int *height);
+/**
+ * Computes the transformed and scaled output resolution.
+ */
 void wlr_output_effective_resolution(struct wlr_output *output,
 	int *width, int *height);
 /**

rootston/output.c

@@ -175,17 +175,6 @@ static bool surface_intersect_output(struct wlr_surface *surface,
 	return wlr_output_layout_intersects(output_layout, wlr_output, &layout_box);
 }
 
-static void output_get_transformed_size(struct wlr_output *wlr_output,
-		int *width, int *height) {
-	if (wlr_output->transform % 2 == 0) {
-		*width = wlr_output->width;
-		*height = wlr_output->height;
-	} else {
-		*width = wlr_output->height;
-		*height = wlr_output->width;
-	}
-}
-
 static void scissor_output(struct roots_output *output, pixman_box32_t *rect) {
 	struct wlr_output *wlr_output = output->wlr_output;
 
@@ -197,7 +186,7 @@ static void scissor_output(struct roots_output *output, pixman_box32_t *rect) {
 	};
 
 	int ow, oh;
-	output_get_transformed_size(output->wlr_output, &ow, &oh);
+	wlr_output_transformed_resolution(output->wlr_output, &ow, &oh);
 
 	// Scissor is in renderer coordinates, ie. upside down
 	enum wl_output_transform transform = wlr_output_transform_compose(
@@ -392,7 +381,7 @@ static void render_output(struct roots_output *output) {
 	}
 
 	int width, height;
-	output_get_transformed_size(output->wlr_output, &width, &height);
+	wlr_output_transformed_resolution(output->wlr_output, &width, &height);
 
 	// Check if we can use damage tracking
 	pixman_region32_t damage;
@@ -506,7 +495,7 @@ static void output_handle_frame(struct wl_listener *listener, void *data) {
 
 static void output_damage_whole(struct roots_output *output) {
 	int width, height;
-	output_get_transformed_size(output->wlr_output, &width, &height);
+	wlr_output_transformed_resolution(output->wlr_output, &width, &height);
 
 	pixman_region32_union_rect(&output->damage, &output->damage, 0, 0,
 		width, height);

types/wlr_output.c

@@ -306,15 +306,20 @@ void wlr_output_destroy(struct wlr_output *output) {
 	}
 }
 
-void wlr_output_effective_resolution(struct wlr_output *output,
+void wlr_output_transformed_resolution(struct wlr_output *output,
 		int *width, int *height) {
-	if (output->transform % 2 == 1) {
-		*width = output->height;
-		*height = output->width;
-	} else {
+	if (output->transform % 2 == 0) {
 		*width = output->width;
 		*height = output->height;
+	} else {
+		*width = output->height;
+		*height = output->width;
 	}
+}
+
+void wlr_output_effective_resolution(struct wlr_output *output,
+		int *width, int *height) {
+	wlr_output_transformed_resolution(output, width, height);
 	*width /= output->scale;
 	*height /= output->scale;
 }
@@ -323,17 +328,6 @@ bool wlr_output_make_current(struct wlr_output *output, int *buffer_age) {
 	return output->impl->make_current(output, buffer_age);
 }
 
-static void output_get_transformed_size(struct wlr_output *output,
-		int *width, int *height) {
-	if (output->transform % 2 == 0) {
-		*width = output->width;
-		*height = output->height;
-	} else {
-		*width = output->height;
-		*height = output->width;
-	}
-}
-
 static void output_scissor(struct wlr_output *output, pixman_box32_t *rect) {
 	struct wlr_box box = {
 		.x = rect->x1,
@@ -343,7 +337,7 @@ static void output_scissor(struct wlr_output *output, pixman_box32_t *rect) {
 	};
 
 	int ow, oh;
-	output_get_transformed_size(output, &ow, &oh);
+	wlr_output_transformed_resolution(output, &ow, &oh);
 
 	// Scissor is in renderer coordinates, ie. upside down
 	enum wl_output_transform transform = wlr_output_transform_compose(
@@ -475,7 +469,7 @@ bool wlr_output_swap_buffers(struct wlr_output *output, struct timespec *when,
 	wl_signal_emit(&output->events.swap_buffers, damage);
 
 	int width, height;
-	output_get_transformed_size(output, &width, &height);
+	wlr_output_transformed_resolution(output, &width, &height);
 
 	pixman_region32_t render_damage;
 	pixman_region32_init(&render_damage);