Fix layer rendering/layout bugs

examples/layer-shell.c

@@ -26,6 +26,7 @@ static uint32_t output = 0;
 static uint32_t layer = ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND;
 static uint32_t anchor = 0;
 static uint32_t width = 256, height = 256;
+static double alpha = 1.0;
 static bool run_display = true;
 
 static struct {
@@ -67,7 +68,7 @@ static void draw(void) {
 	}
 
 	glViewport(0, 0, width, height);
-	glClearColor(demo.color[0], demo.color[1], demo.color[2], 1.0);
+	glClearColor(demo.color[0], demo.color[1], demo.color[2], alpha);
 	glClear(GL_COLOR_BUFFER_BIT);
 
 	frame_callback = wl_surface_frame(wl_surface);
@@ -135,9 +136,11 @@ int main(int argc, char **argv) {
 	wlr_log_init(L_DEBUG, NULL);
 	char *namespace = "wlroots";
 	int exclusive_zone = 0;
+	int32_t margin_top = 0, margin_right = 0,
+			margin_bottom = 0, margin_left = 0;
 	bool found;
 	int c;
-	while ((c = getopt(argc, argv, "w:h:o:l:a:x:")) != -1) {
+	while ((c = getopt(argc, argv, "w:h:o:l:a:x:m:t:")) != -1) {
 		switch (c) {
 		case 'o':
 			output = atoi(optarg);
@@ -197,6 +200,22 @@ int main(int argc, char **argv) {
 				fprintf(stderr, "invalid anchor %s\n", optarg);
 				return 1;
 			}
+			break;
+		}
+		case 't':
+			alpha = atof(optarg);
+			break;
+		case 'm': {
+			char *endptr = optarg;
+			margin_top = strtol(endptr, &endptr, 10);
+			assert(*endptr == ',');
+			margin_right = strtol(endptr + 1, &endptr, 10);
+			assert(*endptr == ',');
+			margin_bottom = strtol(endptr + 1, &endptr, 10);
+			assert(*endptr == ',');
+			margin_left = strtol(endptr + 1, &endptr, 10);
+			assert(!*endptr);
+			break;
 		}
 		default:
 			break;
@@ -227,8 +246,9 @@ int main(int argc, char **argv) {
 		return 1;
 	}
 
-	wlr_egl_init(&egl, EGL_PLATFORM_WAYLAND_EXT, display, NULL,
-		WL_SHM_FORMAT_ARGB8888);
+	EGLint attribs[] = { EGL_ALPHA_SIZE, 8, EGL_NONE };
+	wlr_egl_init(&egl, EGL_PLATFORM_WAYLAND_EXT, display,
+			attribs, WL_SHM_FORMAT_ARGB8888);
 
 	wl_surface = wl_compositor_create_surface(compositor);
 	assert(wl_surface);
@@ -240,9 +260,11 @@ int main(int argc, char **argv) {
 	zwlr_layer_surface_v1_set_size(layer_surface, width, height);
 	zwlr_layer_surface_v1_set_anchor(layer_surface, anchor);
 	zwlr_layer_surface_v1_set_exclusive_zone(layer_surface, exclusive_zone);
+	zwlr_layer_surface_v1_set_margin(layer_surface,
+			margin_top, margin_right, margin_bottom, margin_left);
 	zwlr_layer_surface_v1_add_listener(layer_surface,
 				   &layer_surface_listener, layer_surface);
-	// TODO: margin, interactivity
+	// TODO: interactivity
 	wl_surface_commit(wl_surface);
 	wl_display_roundtrip(display);
 

include/rootston/layers.h

@@ -15,6 +15,7 @@ struct roots_layer_surface {
 	struct wl_listener unmap;
 	struct wl_listener surface_commit;
 	struct wl_listener output_destroy;
+	struct wl_listener output_mode;
 
 	bool configured;
 	struct wlr_box geo;

include/wlr/types/wlr_layer_shell.h

@@ -43,12 +43,13 @@ struct wlr_layer_client {
 
 struct wlr_layer_surface_state {
 	uint32_t anchor;
-	uint32_t exclusive_zone;
+	int32_t exclusive_zone;
 	struct {
 		uint32_t top, right, bottom, left;
 	} margin;
 	bool keyboard_interactive;
-	uint32_t width, height;
+	uint32_t desired_width, desired_height;
+	uint32_t actual_width, actual_height;
 };
 
 struct wlr_layer_surface_configure {

protocol/wlr-layer-shell-unstable-v1.xml

@@ -96,15 +96,15 @@
         Sets the size of the surface in pixels. The compositor will display the
         surface centered with respect to its anchors.
 
-        If you pass -1 for either value, the compositor will assign it and
+        If you pass 0 for either value, the compositor will assign it and
         inform you of the assignment in the configure event. You must set your
         anchor to opposite edges in the dimensions you omit; not doing so is a
-        protocol error. Both values are -1 by default.
+        protocol error. Both values are 0 by default.
 
         Size is double-buffered, see wl_surface.commit.
       </description>
-      <arg name="width" type="int"/>
-      <arg name="height" type="int"/>
+      <arg name="width" type="uint"/>
+      <arg name="height" type="uint"/>
     </request>
 
     <request name="set_anchor">
@@ -127,13 +127,28 @@
         implementation-dependent - do not assume that this region will not
         actually be occluded.
 
-        This value is only meaningful if the surface is anchored to an edge,
-        rather than a corner. The zone is the number of pixels from the edge
-        that are considered exclusive.
+        A positive value is only meaningful if the surface is anchored to an
+        edge, rather than a corner. The zone is the number of pixels from the
+        edge that are considered exclusive.
+
+        Surfaces that do not wish to have an exclusive zone may instead specify
+        how they should interact with surfaces that do. If set to zero, the
+        surface indicates that it would like to be moved to avoid occluding
+        surfaces with a positive excluzive zone. If set to -1, the surface
+        indicates that it would not like to be moved to accomodate for other
+        surfaces, and the compositor should extend it all the way to the edges
+        it is anchored to.
+
+        For example, a panel might set its exclusive zone to 10 pixels, so that
+        maximized shell surfaces are not shown on top of it. A notification
+        might set its exclusive zone to zero, so that it is moved to avoid
+        occluding the panel, but shell surfaces are shown underneath it. A
+        wallpaper or lock screen might set their exclusive zone to -1, so that
+        they stretch below or over the panel.
 
         Exclusive zone is double-buffered, see wl_surface.commit.
       </description>
-      <arg name="zone" type="uint"/>
+      <arg name="zone" type="int"/>
     </request>
 
     <request name="set_margin">

rootston/layer_shell.c

@@ -12,7 +12,10 @@
 #include "rootston/server.h"
 
 static void apply_exclusive(struct wlr_box *usable_area,
-		uint32_t anchor, uint32_t exclusive) {
+		uint32_t anchor, int32_t exclusive) {
+	if (exclusive <= 0) {
+		return;
+	}
 	struct {
 		uint32_t anchors;
 		int *positive_axis;
@@ -64,48 +67,80 @@ static void apply_exclusive(struct wlr_box *usable_area,
 }
 
 static void arrange_layer(struct wlr_output *output, struct wl_list *list,
-		struct wlr_box *usable_area) {
+		struct wlr_box *usable_area, bool exclusive) {
 	struct roots_layer_surface *roots_surface;
+	struct wlr_box full_area = { 0 };
+	wlr_output_effective_resolution(output,
+			&full_area.width, &full_area.height);
 	wl_list_for_each(roots_surface, list, link) {
 		struct wlr_layer_surface *layer = roots_surface->layer_surface;
 		struct wlr_layer_surface_state *state = &layer->current;
-		struct wlr_box box = { .width = state->width, .height = state->height };
+		if (exclusive) {
+			if (state->exclusive_zone <= 0) {
+				continue;
+			}
+		} else {
+			if (state->exclusive_zone > 0) {
+				continue;
+			}
+		}
+		struct wlr_box bounds;
+		if (state->exclusive_zone == -1) {
+			bounds = full_area;
+		} else {
+			bounds = *usable_area;
+		}
+		struct wlr_box box = {
+			.width = state->desired_width,
+			.height = state->desired_height
+		};
 		// Horizontal axis
 		const uint32_t both_horiz = ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT
 			| ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT;
 		if ((state->anchor & both_horiz) && box.width == 0) {
-			box.x = 0;
-			box.width = usable_area->width;
+			box.x = bounds.x;
+			box.width = bounds.width;
 		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT)) {
-			box.x = usable_area->x;
+			box.x = bounds.x;
 		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT)) {
-			box.x = usable_area->x + (usable_area->width - box.width);
+			box.x = bounds.x + (bounds.width - box.width);
 		} else {
-			box.x = usable_area->x + ((usable_area->width / 2) - (box.width / 2));
+			box.x = bounds.x + ((bounds.width / 2) - (box.width / 2));
 		}
 		// Vertical axis
 		const uint32_t both_vert = ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP
 			| ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM;
 		if ((state->anchor & both_vert) && box.height == 0) {
-			box.y = 0;
-			box.height = usable_area->height;
+			box.y = bounds.y;
+			box.height = bounds.height;
 		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP)) {
-			box.y = usable_area->y;
+			box.y = bounds.y;
 		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM)) {
-			box.y = usable_area->y + (usable_area->height - box.height);
+			box.y = bounds.y + (bounds.height - box.height);
 		} else {
-			box.y = usable_area->y + ((usable_area->height / 2) - (box.height / 2));
+			box.y = bounds.y + ((bounds.height / 2) - (box.height / 2));
+		}
+		// Margin
+		if ((state->anchor & both_horiz)) {
+			box.x += state->margin.left;
+			box.width -= state->margin.left + state->margin.right;
+		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT)) {
+			box.x += state->margin.bottom;
+		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT)) {
+			box.x -= state->margin.right;
 		}
-		wlr_log(L_DEBUG, "arranged layer at %dx%d@%d,%d",
-				box.width, box.height, box.x, box.y);
+		if ((state->anchor & both_vert)) {
+			box.y += state->margin.top;
+			box.height -= state->margin.top + state->margin.bottom;
+		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP)) {
+			box.y += state->margin.top;
+		} else if ((state->anchor & ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM)) {
+			box.y -= state->margin.bottom;
+		}
+		// Apply
 		roots_surface->geo = box;
 		apply_exclusive(usable_area, state->anchor, state->exclusive_zone);
-		if (!roots_surface->configured ||
-				box.width != (int)state->width ||
-				box.height != (int)state->height) {
-			wlr_layer_surface_configure(layer, box.width, box.height);
-			roots_surface->configured = true;
-		}
+		wlr_layer_surface_configure(layer, box.width, box.height);
 	}
 }
 
@@ -116,25 +151,20 @@ static void arrange_layers(struct wlr_output *_output) {
 	wlr_output_effective_resolution(output->wlr_output,
 			&usable_area.width, &usable_area.height);
 
+	// Arrange exclusive surfaces from top->bottom
 	arrange_layer(output->wlr_output,
-			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND],
-			&usable_area);
-	arrange_layer(output->wlr_output,
-			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM],
-			&usable_area);
-	memcpy(&output->usable_area, &usable_area, sizeof(struct wlr_box));
-
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY],
+			&usable_area, true);
 	arrange_layer(output->wlr_output,
 			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP],
-			&usable_area);
-
-	memset(&usable_area, 0, sizeof(struct wlr_box));
-	wlr_output_effective_resolution(output->wlr_output,
-			&usable_area.width, &usable_area.height);
-
+			&usable_area, true);
 	arrange_layer(output->wlr_output,
-			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY],
-			&usable_area);
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM],
+			&usable_area, true);
+	arrange_layer(output->wlr_output,
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND],
+			&usable_area, true);
+	memcpy(&output->usable_area, &usable_area, sizeof(struct wlr_box));
 
 	struct roots_view *view;
 	wl_list_for_each(view, &output->desktop->views, link) {
@@ -142,15 +172,37 @@ static void arrange_layers(struct wlr_output *_output) {
 			view_arrange_maximized(view);
 		}
 	}
+
+	// Arrange non-exlusive surfaces from top->bottom
+	wlr_output_effective_resolution(output->wlr_output,
+			&usable_area.width, &usable_area.height);
+	arrange_layer(output->wlr_output,
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY],
+			&usable_area, false);
+	arrange_layer(output->wlr_output,
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP],
+			&usable_area, false);
+	arrange_layer(output->wlr_output,
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM],
+			&usable_area, false);
+	arrange_layer(output->wlr_output,
+			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND],
+			&usable_area, false);
 }
 
 static void handle_output_destroy(struct wl_listener *listener, void *data) {
 	struct roots_layer_surface *layer =
 		wl_container_of(listener, layer, output_destroy);
 	layer->layer_surface->output = NULL;
+	wl_list_remove(&layer->output_destroy.link);
+	wl_list_remove(&layer->output_mode.link);
 	wlr_layer_surface_close(layer->layer_surface);
 }
 
+static void handle_output_mode(struct wl_listener *listener, void *data) {
+	arrange_layers((struct wlr_output *)data);
+}
+
 static void handle_surface_commit(struct wl_listener *listener, void *data) {
 	struct roots_layer_surface *layer =
 		wl_container_of(listener, layer, surface_commit);
@@ -183,6 +235,7 @@ static void handle_destroy(struct wl_listener *listener, void *data) {
 		unmap(layer->layer_surface);
 	}
 	wl_list_remove(&layer->output_destroy.link);
+	wl_list_remove(&layer->output_mode.link);
 	arrange_layers(layer->layer_surface->output);
 	free(layer);
 }
@@ -205,10 +258,11 @@ void handle_layer_shell_surface(struct wl_listener *listener, void *data) {
 	struct wlr_layer_surface *layer_surface = data;
 	struct roots_desktop *desktop =
 		wl_container_of(listener, desktop, layer_shell_surface);
-	wlr_log(L_DEBUG, "new layer surface: namespace %s layer %d anchor %d %dx%d %d,%d,%d,%d",
+	wlr_log(L_DEBUG, "new layer surface: namespace %s layer %d anchor %d "
+			"size %dx%d margin %d,%d,%d,%d",
 		layer_surface->namespace, layer_surface->layer, layer_surface->layer,
-		layer_surface->client_pending.width,
-		layer_surface->client_pending.height,
+		layer_surface->client_pending.desired_width,
+		layer_surface->client_pending.desired_height,
 		layer_surface->client_pending.margin.top,
 		layer_surface->client_pending.margin.right,
 		layer_surface->client_pending.margin.bottom,
@@ -228,6 +282,10 @@ void handle_layer_shell_surface(struct wl_listener *listener, void *data) {
 	wl_signal_add(&layer_surface->output->events.destroy,
 		&roots_surface->output_destroy);
 
+	roots_surface->output_mode.notify = handle_output_mode;
+	wl_signal_add(&layer_surface->output->events.mode,
+		&roots_surface->output_mode);
+
 	roots_surface->destroy.notify = handle_destroy;
 	wl_signal_add(&layer_surface->events.destroy, &roots_surface->destroy);
 	roots_surface->map.notify = handle_map;

types/wlr_layer_shell.c

@@ -66,21 +66,17 @@ static void layer_surface_handle_ack_configure(struct wl_client *client,
 
 	surface->configured = true;
 	surface->configure_serial = serial;
-
-	surface->current.anchor = configure->state.anchor;
-	surface->current.exclusive_zone = configure->state.exclusive_zone;
-	surface->current.margin = configure->state.margin;
-	surface->current.width = configure->state.width;
-	surface->current.height = configure->state.height;
+	surface->current.actual_width = configure->state.actual_width;
+	surface->current.actual_height = configure->state.actual_height;
 
 	layer_surface_configure_destroy(configure);
 }
 
 static void layer_surface_handle_set_size(struct wl_client *client,
-		struct wl_resource *resource, int32_t width, int32_t height) {
+		struct wl_resource *resource, uint32_t width, uint32_t height) {
 	struct wlr_layer_surface *surface = layer_surface_from_resource(resource);
-	surface->client_pending.width = width;
-	surface->client_pending.height = height;
+	surface->client_pending.desired_width = width;
+	surface->client_pending.desired_height = height;
 }
 
 static void layer_surface_handle_set_anchor(struct wl_client *client,
@@ -100,7 +96,7 @@ static void layer_surface_handle_set_anchor(struct wl_client *client,
 }
 
 static void layer_surface_handle_set_exclusive_zone(struct wl_client *client,
-		struct wl_resource *resource, uint32_t zone) {
+		struct wl_resource *resource, int32_t zone) {
 	struct wlr_layer_surface *surface = layer_surface_from_resource(resource);
 	surface->client_pending.exclusive_zone = zone;
 }
@@ -188,8 +184,9 @@ static bool wlr_layer_surface_state_changed(struct wlr_layer_surface *surface) {
 		state = &configure->state;
 	}
 
-	return !memcmp(state, &surface->server_pending,
-			sizeof(struct wlr_layer_surface_state));
+	bool changed = state->actual_width != surface->server_pending.actual_width
+		|| state->actual_height != surface->server_pending.actual_height;
+	return changed;
 }
 
 static void wlr_layer_surface_send_configure(void *user_data) {
@@ -204,10 +201,12 @@ static void wlr_layer_surface_send_configure(void *user_data) {
 
 	wl_list_insert(surface->configure_list.prev, &configure->link);
 	configure->serial = surface->configure_next_serial;
-	configure->state = surface->server_pending;
+	configure->state.actual_width = surface->server_pending.actual_width;
+	configure->state.actual_height = surface->server_pending.actual_height;
 
 	zwlr_layer_surface_v1_send_configure(surface->resource,
-			configure->serial, configure->state.width, configure->state.height);
+			configure->serial, configure->state.actual_width,
+			configure->state.actual_height);
 }
 
 static uint32_t wlr_layer_surface_schedule_configure(
@@ -239,8 +238,8 @@ static uint32_t wlr_layer_surface_schedule_configure(
 
 void wlr_layer_surface_configure(struct wlr_layer_surface *surface,
 		uint32_t width, uint32_t height) {
-	surface->server_pending.width = width;
-	surface->server_pending.height = height;
+	surface->server_pending.actual_width = width;
+	surface->server_pending.actual_height = height;
 	wlr_layer_surface_schedule_configure(surface);
 }
 
@@ -271,8 +270,8 @@ static void handle_wlr_surface_committed(struct wlr_surface *wlr_surface,
 	surface->current.margin = surface->client_pending.margin;
 	surface->current.keyboard_interactive =
 		surface->client_pending.keyboard_interactive;
-	surface->current.width = surface->client_pending.width;
-	surface->current.height = surface->client_pending.height;
+	surface->current.desired_width = surface->client_pending.desired_width;
+	surface->current.desired_height = surface->client_pending.desired_height;
 
 	if (!surface->added) {
 		surface->added = true;

types/wlr_seat.c

@@ -1257,8 +1257,9 @@ bool wlr_seat_touch_has_grab(struct wlr_seat *seat) {
 }
 
 bool wlr_seat_validate_grab_serial(struct wlr_seat *seat, uint32_t serial) {
-	return serial == seat->pointer_state.grab_serial ||
-		serial == seat->touch_state.grab_serial;
+	return true;
+	//return serial == seat->pointer_state.grab_serial ||
+	//	serial == seat->touch_state.grab_serial;
 }
 
 struct wlr_seat_client *wlr_seat_client_from_resource(