Merge pull request #522 from emersion/fix-drm-output-transform-cursor

Fix cursor on transformed output with DRM backend

backend/drm/drm.c

@@ -543,10 +543,11 @@ static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 
 		// OpenGL will read the pixels out upside down,
 		// so we need to flip the image vertically
+		enum wl_output_transform transform = wlr_output_transform_compose(
+			wlr_output_transform_invert(output->transform),
+			WL_OUTPUT_TRANSFORM_FLIPPED_180);
 		wlr_matrix_texture(plane->matrix, plane->surf.width, plane->surf.height,
-			conn->output.transform ^ WL_OUTPUT_TRANSFORM_FLIPPED_180);
+			transform);
-
-		// TODO the image needs to be rotated depending on the output rotation
 
 		plane->wlr_tex =
 			wlr_render_texture_create(plane->surf.renderer->wlr_rend);

include/wlr/types/wlr_output.h

@@ -118,6 +118,19 @@ bool wlr_output_cursor_move(struct wlr_output_cursor *cursor,
 	double x, double y);
 void wlr_output_cursor_destroy(struct wlr_output_cursor *cursor);
 
-enum wl_output_transform wlr_output_transform_invert(enum wl_output_transform);
+
+/**
+ * Returns the transform that, when composed with `tr`, gives
+ * `WL_OUTPUT_TRANSFORM_NORMAL`.
+ */
+enum wl_output_transform wlr_output_transform_invert(
+	enum wl_output_transform tr);
+
+/**
+ * Returns a transform that, when applied, has the same effect as applying
+ * sequentially `tr_a` and `tr_b`.
+ */
+enum wl_output_transform wlr_output_transform_compose(
+	enum wl_output_transform tr_a, enum wl_output_transform tr_b);
 
 #endif

types/wlr_output.c

@@ -726,11 +726,22 @@ void wlr_output_cursor_destroy(struct wlr_output_cursor *cursor) {
 	free(cursor);
 }
 
+
 enum wl_output_transform wlr_output_transform_invert(
-		enum wl_output_transform transform) {
+		enum wl_output_transform tr) {
-	if ((transform & WL_OUTPUT_TRANSFORM_90) &&
+	if ((tr & WL_OUTPUT_TRANSFORM_90) && !(tr & WL_OUTPUT_TRANSFORM_FLIPPED)) {
-			!(transform & WL_OUTPUT_TRANSFORM_FLIPPED)) {
+		tr ^= WL_OUTPUT_TRANSFORM_180;
-		transform ^= WL_OUTPUT_TRANSFORM_180;
+	}
+	return tr;
+}
+
+enum wl_output_transform wlr_output_transform_compose(
+		enum wl_output_transform tr_a, enum wl_output_transform tr_b) {
+	uint32_t flipped = (tr_a ^ tr_b) & WL_OUTPUT_TRANSFORM_FLIPPED;
+	uint32_t rotated =
+		(tr_a + tr_b) & (WL_OUTPUT_TRANSFORM_90 | WL_OUTPUT_TRANSFORM_180);
+	if ((tr_a & WL_OUTPUT_TRANSFORM_FLIPPED) && (tr_b & WL_OUTPUT_TRANSFORM_FLIPPED)) {
+		rotated = wlr_output_transform_invert(rotated);
 	}
-	return transform;
+	return flipped | rotated;
 }