output-damage: fix damage on modeset

On modeset wlr_output will internally allocate a buffer. The
backend will emit a "mode" output event, then wlr_output will
emit a "commit" event.

wlr_output_damage handles the "mode" event by damaging the whole
output, and then handles the "commit" event. However the commit
event has a buffer, so wlr_output_damage rotates the damage in its
ring buffer, thinking the compositor has rendered a frame. The
compositor hasn't rendered a frame, what wlr_output_damage sees is
the internal wlr_output black buffer used for the modeset.

Let's fix this by damaging the whole output in the "commit" event
handler if the mode has changed. Additionally, damage the whole
output after rotating the damage ring buffer.

types/wlr_output_damage.c

@@ -61,31 +61,30 @@ static void output_handle_commit(struct wl_listener *listener, void *data) {
 		wl_container_of(listener, output_damage, output_commit);
 	struct wlr_output_event_commit *event = data;
 
-	if (event->committed & (WLR_OUTPUT_STATE_SCALE | WLR_OUTPUT_STATE_TRANSFORM)) {
+	if (event->committed & WLR_OUTPUT_STATE_BUFFER) {
-		wlr_output_damage_add_whole(output_damage);
+		pixman_region32_t *prev;
-	}
+		if (output_damage->pending_attach_render) {
+			// render-buffers have been swapped, rotate the damage
+
+			// same as decrementing, but works on unsigned integers
+			output_damage->previous_idx += WLR_OUTPUT_DAMAGE_PREVIOUS_LEN - 1;
+			output_damage->previous_idx %= WLR_OUTPUT_DAMAGE_PREVIOUS_LEN;
+
+			prev = &output_damage->previous[output_damage->previous_idx];
+			pixman_region32_copy(prev, &output_damage->current);
+		} else {
+			// accumulate render-buffer damage
+			prev = &output_damage->previous[output_damage->previous_idx];
+			pixman_region32_union(prev, prev, &output_damage->current);
+		}
 
-	if (!(event->committed & WLR_OUTPUT_STATE_BUFFER)) {
+		pixman_region32_clear(&output_damage->current);
-		return;
 	}
 
-	pixman_region32_t *prev;
+	if (event->committed & (WLR_OUTPUT_STATE_MODE | WLR_OUTPUT_STATE_SCALE |
-	if (output_damage->pending_attach_render) {
+			WLR_OUTPUT_STATE_TRANSFORM)) {
-		// render-buffers have been swapped, rotate the damage
+		wlr_output_damage_add_whole(output_damage);
-
-		// same as decrementing, but works on unsigned integers
-		output_damage->previous_idx += WLR_OUTPUT_DAMAGE_PREVIOUS_LEN - 1;
-		output_damage->previous_idx %= WLR_OUTPUT_DAMAGE_PREVIOUS_LEN;
-
-		prev = &output_damage->previous[output_damage->previous_idx];
-		pixman_region32_copy(prev, &output_damage->current);
-	} else {
-		// accumulate render-buffer damage
-		prev = &output_damage->previous[output_damage->previous_idx];
-		pixman_region32_union(prev, prev, &output_damage->current);
 	}
-
-	pixman_region32_clear(&output_damage->current);
 }
 
 struct wlr_output_damage *wlr_output_damage_create(struct wlr_output *output) {