layer_shell: damage previous area when a surface shrinks

When a layer surface shrinks we need to damage the area it previously
occupied, but we don't know the location of all its subsurfaces in the
previous state, so instead damage a rectangle that encloses the entire
previous extent.

include/sway/layers.h

@@ -22,6 +22,7 @@ struct sway_layer_surface {
 	struct wl_listener new_subsurface;
 
 	struct wlr_box geo;
+	struct wlr_box extent;
 	enum zwlr_layer_shell_v1_layer layer;
 };
 

sway/desktop/layer_shell.c

@@ -169,6 +169,9 @@ static void arrange_layer(struct sway_output *output, struct wl_list *list,
 		}
 		// Apply
 		sway_layer->geo = box;
+		wlr_surface_get_extends(layer->surface, &sway_layer->extent);
+		sway_layer->extent.x += box.x;
+		sway_layer->extent.y += box.y;
 		apply_exclusive(usable_area, state->anchor, state->exclusive_zone,
 				state->margin.top, state->margin.right,
 				state->margin.bottom, state->margin.left);
@@ -297,11 +300,11 @@ static void handle_surface_commit(struct wl_listener *listener, void *data) {
 	}
 
 	struct sway_output *output = wlr_output->data;
-	struct wlr_box old_geo = layer->geo;
+	struct wlr_box old_extent = layer->extent;
 	arrange_layers(output);
 
-	bool geo_changed =
-		memcmp(&old_geo, &layer->geo, sizeof(struct wlr_box)) != 0;
+	bool extent_changed =
+		memcmp(&old_extent, &layer->extent, sizeof(struct wlr_box)) != 0;
 	bool layer_changed = layer->layer != layer_surface->current.layer;
 	if (layer_changed) {
 		wl_list_remove(&layer->link);
@@ -309,9 +312,8 @@ static void handle_surface_commit(struct wl_listener *listener, void *data) {
 			&layer->link);
 		layer->layer = layer_surface->current.layer;
 	}
-	if (geo_changed || layer_changed) {
-		output_damage_surface(output, old_geo.x, old_geo.y,
-			layer_surface->surface, true);
+	if (extent_changed || layer_changed) {
+		output_damage_box(output, &old_extent);
 		output_damage_surface(output, layer->geo.x, layer->geo.y,
 			layer_surface->surface, true);
 	} else {