Fix damage tracking for rotated surfaces

It was broken because the damage extents were rotated about its
own center, not about the center of the surface.

This adds a new wlr_region_rotated_bounds that rotates regions.
This allows us to have only one code path (for both non-rotated
views and rotated views) and optimizes rendering for rotated
views.

include/wlr/types/wlr_box.h

@@ -27,7 +27,7 @@ void wlr_box_transform(const struct wlr_box *box,
 	struct wlr_box *dest);
 
 /**
- * Creates the smallest box that contains a rotated box.
+ * Creates the smallest box that contains the box rotated about its center.
  */
 void wlr_box_rotated_bounds(const struct wlr_box *box, float rotation,
 	struct wlr_box *dest);

include/wlr/util/region.h

@@ -26,4 +26,11 @@ void wlr_region_transform(pixman_region32_t *dst, pixman_region32_t *src,
 void wlr_region_expand(pixman_region32_t *dst, pixman_region32_t *src,
 	int distance);
 
+/*
+ * Builds the smallest possible region that contains the region rotated about
+ * the point (ox, oy).
+ */
+void wlr_region_rotated_bounds(pixman_region32_t *dst, pixman_region32_t *src,
+	float rotation, int ox, int oy);
+
 #endif

rootston/output.c

@@ -486,6 +486,9 @@ static void render_output(struct roots_output *output) {
 		goto renderer_end;
 	}
 
+	// Uncomment this line to debug damage tracking
+	//wlr_renderer_clear(renderer, (float[]){1, 1, 0, 0});
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
@@ -667,32 +670,23 @@ static void damage_from_surface(struct wlr_surface *surface,
 	surface_intersect_output(surface, output->desktop->layout,
 		wlr_output, lx, ly, rotation, &box);
 
-	if (rotation == 0) {
-		pixman_region32_t damage;
-		pixman_region32_init(&damage);
-		pixman_region32_copy(&damage, &surface->current->surface_damage);
-		wlr_region_scale(&damage, &damage, wlr_output->scale);
-		if (ceil(wlr_output->scale) > surface->current->scale) {
-			// When scaling up a surface, it'll become blurry so we need to
-			// expand the damage region
-			wlr_region_expand(&damage, &damage,
-				ceil(wlr_output->scale) - surface->current->scale);
-		}
-		pixman_region32_translate(&damage, box.x, box.y);
-		wlr_output_damage_add(output->damage, &damage);
-		pixman_region32_fini(&damage);
-	} else {
-		pixman_box32_t *extents =
-			pixman_region32_extents(&surface->current->surface_damage);
-		struct wlr_box damage_box = {
-			.x = box.x + extents->x1 * wlr_output->scale,
-			.y = box.y + extents->y1 * wlr_output->scale,
-			.width = (extents->x2 - extents->x1) * wlr_output->scale,
-			.height = (extents->y2 - extents->y1) * wlr_output->scale,
-		};
-		wlr_box_rotated_bounds(&damage_box, rotation, &damage_box);
-		wlr_output_damage_add_box(output->damage, &damage_box);
-	}
+	int center_x = box.x + box.width/2;
+	int center_y = box.y + box.height/2;
+
+	pixman_region32_t damage;
+	pixman_region32_init(&damage);
+	pixman_region32_copy(&damage, &surface->current->surface_damage);
+	wlr_region_scale(&damage, &damage, wlr_output->scale);
+	if (ceil(wlr_output->scale) > surface->current->scale) {
+		// When scaling up a surface, it'll become blurry so we need to
+		// expand the damage region
+		wlr_region_expand(&damage, &damage,
+			ceil(wlr_output->scale) - surface->current->scale);
+	}
+	pixman_region32_translate(&damage, box.x, box.y);
+	wlr_region_rotated_bounds(&damage, &damage, rotation, center_x, center_y);
+	wlr_output_damage_add(output->damage, &damage);
+	pixman_region32_fini(&damage);
 }
 
 void output_damage_from_view(struct roots_output *output,

util/region.c

@@ -128,3 +128,52 @@ void wlr_region_expand(pixman_region32_t *dst, pixman_region32_t *src,
 	pixman_region32_init_rects(dst, dst_rects, nrects);
 	free(dst_rects);
 }
+
+void wlr_region_rotated_bounds(pixman_region32_t *dst, pixman_region32_t *src,
+		float rotation, int ox, int oy) {
+	if (rotation == 0) {
+		pixman_region32_copy(dst, src);
+		return;
+	}
+
+	int nrects;
+	pixman_box32_t *src_rects = pixman_region32_rectangles(src, &nrects);
+
+	pixman_box32_t *dst_rects = malloc(nrects * sizeof(pixman_box32_t));
+	if (dst_rects == NULL) {
+		return;
+	}
+
+	for (int i = 0; i < nrects; ++i) {
+		double x1 = src_rects[i].x1 - ox;
+		double y1 = src_rects[i].y1 - oy;
+		double x2 = src_rects[i].x2 - ox;
+		double y2 = src_rects[i].y2 - oy;
+
+		double rx1 = x1 * cos(rotation) - y1 * sin(rotation);
+		double ry1 = x1 * sin(rotation) + y1 * cos(rotation);
+
+		double rx2 = x2 * cos(rotation) - y1 * sin(rotation);
+		double ry2 = x2 * sin(rotation) + y1 * cos(rotation);
+
+		double rx3 = x2 * cos(rotation) - y2 * sin(rotation);
+		double ry3 = x2 * sin(rotation) + y2 * cos(rotation);
+
+		double rx4 = x1 * cos(rotation) - y2 * sin(rotation);
+		double ry4 = x1 * sin(rotation) + y2 * cos(rotation);
+
+		x1 = fmin(fmin(rx1, rx2), fmin(rx3, rx4));
+		y1 = fmin(fmin(ry1, ry2), fmin(ry3, ry4));
+		x2 = fmax(fmax(rx1, rx2), fmax(rx3, rx4));
+		y2 = fmax(fmax(ry1, ry2), fmax(ry3, ry4));
+
+		dst_rects[i].x1 = floor(ox + x1);
+		dst_rects[i].x2 = ceil(ox + x2);
+		dst_rects[i].y1 = floor(oy + y1);
+		dst_rects[i].y2 = ceil(oy + y2);
+	}
+
+	pixman_region32_fini(dst);
+	pixman_region32_init_rects(dst, dst_rects, nrects);
+	free(dst_rects);
+}