Fix hidden software cursors, fix cursor transformations on DRM backend

backend/drm/drm.c

@@ -558,17 +558,28 @@ static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 		}
 	}
 
-	struct wlr_box hotspot_box = {
-		.width = plane->surf.width,
-		.height = plane->surf.height,
-		.x = hotspot_x,
-		.y = hotspot_y,
-	};
-	struct wlr_box transformed_hotspot_box;
-	wlr_output_transform_apply_to_box(output->transform,
-		&hotspot_box, &transformed_hotspot_box);
-	plane->cursor_hotspot_x = transformed_hotspot_box.x;
-	plane->cursor_hotspot_y = transformed_hotspot_box.y;
+	switch (output->transform) {
+	case WL_OUTPUT_TRANSFORM_NORMAL:
+	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+		plane->cursor_hotspot_x = hotspot_x;
+		plane->cursor_hotspot_y = hotspot_y;
+		break;
+	case WL_OUTPUT_TRANSFORM_90:
+	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+		plane->cursor_hotspot_x = hotspot_x;
+		plane->cursor_hotspot_y = -plane->surf.height + hotspot_y;
+		break;
+	case WL_OUTPUT_TRANSFORM_180:
+	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+		plane->cursor_hotspot_x = -plane->surf.width + hotspot_x;
+		plane->cursor_hotspot_y = -plane->surf.height + hotspot_y;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED:
+	case WL_OUTPUT_TRANSFORM_270:
+		plane->cursor_hotspot_x = -plane->surf.width + hotspot_x;
+		plane->cursor_hotspot_y = hotspot_y;
+		break;
+	}
 
 	if (!update_pixels) {
 		// Only update the cursor hotspot
@@ -621,30 +632,18 @@ static bool wlr_drm_connector_move_cursor(struct wlr_output *output,
 	x -= plane->cursor_hotspot_x;
 	y -= plane->cursor_hotspot_y;
 
-	int width, height, tmp;
-	wlr_output_effective_resolution(output, &width, &height);
+	struct wlr_box box;
+	box.x = x;
+	box.y = y;
+	wlr_output_effective_resolution(output, &box.width, &box.height);
 
-	switch (output->transform) {
-	case WL_OUTPUT_TRANSFORM_NORMAL:
-	case WL_OUTPUT_TRANSFORM_FLIPPED:
-	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
-		// nothing to do
-		break;
-	case WL_OUTPUT_TRANSFORM_270:
-	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
-		tmp = x;
-		x = y;
-		y = -(tmp - width);
-		break;
-	case WL_OUTPUT_TRANSFORM_90:
-	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
-		tmp = x;
-		x = -(y - height);
-		y = tmp;
-		break;
-	}
+	enum wl_output_transform transform =
+		wlr_output_transform_invert(output->transform);
+	struct wlr_box transformed_box;
+	wlr_output_transform_apply_to_box(transform, &box, &transformed_box);
 
-	return drm->iface->crtc_move_cursor(drm, conn->crtc, x, y);
+	return drm->iface->crtc_move_cursor(drm, conn->crtc, transformed_box.x,
+		transformed_box.y);
 }
 
 static void wlr_drm_connector_destroy(struct wlr_output *output) {

backend/wayland/output.c

@@ -59,6 +59,7 @@ static bool wlr_wl_output_set_cursor(struct wlr_output *_output,
 		(struct wlr_wl_backend_output *)_output;
 	struct wlr_wl_backend *backend = output->backend;
 
+	// TODO: use output->wlr_output.transform to transform pixels and hotpot
 	output->cursor.hotspot_x = hotspot_x;
 	output->cursor.hotspot_y = hotspot_y;
 

include/wlr/interfaces/wlr_output.h

@@ -34,5 +34,6 @@ void wlr_output_destroy_global(struct wlr_output *wlr_output);
 
 void wlr_output_transform_apply_to_box(enum wl_output_transform transform,
 	struct wlr_box *box, struct wlr_box *dest);
+enum wl_output_transform wlr_output_transform_invert(enum wl_output_transform);
 
 #endif

rootston/config.c

@@ -228,7 +228,9 @@ static int config_ini_handler(void *user, const char *section, const char *name,
 		} else if (strcmp(name, "y") == 0) {
 			oc->y = strtol(value, NULL, 10);
 		} else if (strcmp(name, "rotate") == 0) {
-			if (strcmp(value, "90") == 0) {
+			if (strcmp(value, "normal") == 0) {
+				oc->transform = WL_OUTPUT_TRANSFORM_NORMAL;
+			} else if (strcmp(value, "90") == 0) {
 				oc->transform = WL_OUTPUT_TRANSFORM_90;
 			} else if (strcmp(value, "180") == 0) {
 				oc->transform = WL_OUTPUT_TRANSFORM_180;

types/wlr_output.c

@@ -267,8 +267,8 @@ static void output_cursor_render(struct wlr_output_cursor *cursor) {
 
 	struct wlr_box output_box;
 	output_box.x = output_box.y = 0;
-	output_box.width = cursor->output->width;
-	output_box.height = cursor->output->height;
+	wlr_output_effective_resolution(cursor->output, &output_box.width,
+		&output_box.height);
 
 	struct wlr_box cursor_box;
 	output_cursor_get_box(cursor, &cursor_box);
@@ -294,7 +294,7 @@ void wlr_output_swap_buffers(struct wlr_output *output) {
 	struct wlr_output_cursor *cursor;
 	wl_list_for_each(cursor, &output->cursors, link) {
 		if (output->hardware_cursor == cursor) {
-			continue;
+			//continue; // TODO
 		}
 		output_cursor_render(cursor);
 	}
@@ -342,7 +342,7 @@ bool wlr_output_cursor_set_image(struct wlr_output_cursor *cursor,
 			stride, width, height, hotspot_x, hotspot_y, true);
 		if (ok) {
 			cursor->output->hardware_cursor = cursor;
-			return true;
+			//return true; // TODO
 		}
 	}
 
@@ -546,3 +546,12 @@ void wlr_output_transform_apply_to_box(enum wl_output_transform transform,
 		break;
 	}
 }
+
+enum wl_output_transform wlr_output_transform_invert(
+		enum wl_output_transform transform) {
+	if ((transform & WL_OUTPUT_TRANSFORM_90) &&
+			!(transform & WL_OUTPUT_TRANSFORM_FLIPPED)) {
+		transform ^= WL_OUTPUT_TRANSFORM_180;
+	}
+	return transform;
+}