output: fix output transform compositions

This change ensures that wlr_output_transform_compose correctly composes
transforms when the first transform includes a rotation and the second
transform includes a flip.

types/wlr_output.c

@@ -1118,7 +1118,15 @@ enum wl_output_transform wlr_output_transform_invert(
 enum wl_output_transform wlr_output_transform_compose(
 		enum wl_output_transform tr_a, enum wl_output_transform tr_b) {
 	uint32_t flipped = (tr_a ^ tr_b) & WL_OUTPUT_TRANSFORM_FLIPPED;
-	uint32_t rotated =
-		(tr_a + tr_b) & (WL_OUTPUT_TRANSFORM_90 | WL_OUTPUT_TRANSFORM_180);
+	uint32_t rotation_mask = WL_OUTPUT_TRANSFORM_90 | WL_OUTPUT_TRANSFORM_180;
+	uint32_t rotated;
+	if (tr_b & WL_OUTPUT_TRANSFORM_FLIPPED) {
+		// When a rotation of k degrees is followed by a flip, the
+		// equivalent transform is a flip followed by a rotation of
+		// -k degrees.
+		rotated = (tr_b - tr_a) & rotation_mask;
+	} else {
+		rotated = (tr_a + tr_b) & rotation_mask;
+	}
 	return flipped | rotated;
 }