Fix popups

Fixes the render and container_at order for popups.

Fixes #2210

For rendering:

* render_view_surfaces has been renamed to render_view_toplevels
* render_view_toplevels now uses output_surface_for_each_surface (which
is now public), as that function uses wlr_surface_for_each_surface which
doesn't descend into popups
* Views now have a for_each_popup iterator, which is used by the
renderer to render the focused view's popups
* When rendering a popup, toplevels (xdg subsurfaces) of that popup are
also rendered

For sending frame done, the logic has been updated to match the
rendering logic:

* send_frame_done_container no longer descends into popups
* for_each_popup is used to send frame done to the focused view's popups
and their child toplevels

For container_at:

* floating_container_at is now static, which means it had to be moved
higher in the file.
* container_at now considers popups for the focused view before checking
containers.
* tiling_container_at has been introduced, so that it doesn't call
container_at recursively (it would check popups recursively if it did)

include/sway/output.h

@@ -67,10 +67,18 @@ struct sway_container *output_get_active_workspace(struct sway_output *output);
 void output_render(struct sway_output *output, struct timespec *when,
 	pixman_region32_t *damage);
 
+void output_surface_for_each_surface(struct sway_output *output,
+		struct wlr_surface *surface, double ox, double oy,
+		sway_surface_iterator_func_t iterator, void *user_data);
+
 void output_view_for_each_surface(struct sway_output *output,
 	struct sway_view *view, sway_surface_iterator_func_t iterator,
 	void *user_data);
 
+void output_view_for_each_popup(struct sway_output *output,
+		struct sway_view *view, sway_surface_iterator_func_t iterator,
+		void *user_data);
+
 void output_layer_for_each_surface(struct sway_output *output,
 	struct wl_list *layer_surfaces, sway_surface_iterator_func_t iterator,
 	void *user_data);

include/sway/tree/container.h

@@ -230,17 +230,10 @@ struct sway_container *container_parent(struct sway_container *container,
  * surface-local coordinates of the given layout coordinates if the container
  * is a view and the view contains a surface at those coordinates.
  */
-struct sway_container *container_at(struct sway_container *container,
+struct sway_container *container_at(struct sway_container *workspace,
-		double ox, double oy, struct wlr_surface **surface,
+		double lx, double ly, struct wlr_surface **surface,
 		double *sx, double *sy);
 
-/**
- * Same as container_at, but only checks floating views and expects coordinates
- * to be layout coordinates, as that's what floating views use.
- */
-struct sway_container *floating_container_at(double lx, double ly,
-		struct wlr_surface **surface, double *sx, double *sy);
-
 /**
  * Apply the function for each descendant of the container breadth first.
  */

include/sway/tree/view.h

@@ -47,6 +47,8 @@ struct sway_view_impl {
 	bool (*has_client_side_decorations)(struct sway_view *view);
 	void (*for_each_surface)(struct sway_view *view,
 		wlr_surface_iterator_func_t iterator, void *user_data);
+	void (*for_each_popup)(struct sway_view *view,
+		wlr_surface_iterator_func_t iterator, void *user_data);
 	void (*close)(struct sway_view *view);
 	void (*destroy)(struct sway_view *view);
 };
@@ -248,9 +250,18 @@ void view_close(struct sway_view *view);
 
 void view_damage_from(struct sway_view *view);
 
+/**
+ * Iterate all surfaces of a view (toplevels + popups).
+ */
 void view_for_each_surface(struct sway_view *view,
 	wlr_surface_iterator_func_t iterator, void *user_data);
 
+/**
+ * Iterate all popups recursively.
+ */
+void view_for_each_popup(struct sway_view *view,
+	wlr_surface_iterator_func_t iterator, void *user_data);
+
 // view implementation
 
 void view_init(struct sway_view *view, enum sway_view_type type,

sway/desktop/output.c

@@ -119,7 +119,7 @@ static void output_for_each_surface_iterator(struct wlr_surface *surface,
 		data->user_data);
 }
 
-static void output_surface_for_each_surface(struct sway_output *output,
+void output_surface_for_each_surface(struct sway_output *output,
 		struct wlr_surface *surface, double ox, double oy,
 		sway_surface_iterator_func_t iterator, void *user_data) {
 	struct surface_iterator_data data = {
@@ -155,6 +155,23 @@ void output_view_for_each_surface(struct sway_output *output,
 		output_for_each_surface_iterator, &data);
 }
 
+void output_view_for_each_popup(struct sway_output *output,
+		struct sway_view *view, sway_surface_iterator_func_t iterator,
+		void *user_data) {
+	struct surface_iterator_data data = {
+		.user_iterator = iterator,
+		.user_data = user_data,
+		.output = output,
+		.ox = view->swayc->current.view_x - output->swayc->current.swayc_x,
+		.oy = view->swayc->current.view_y - output->swayc->current.swayc_y,
+		.width = view->swayc->current.view_width,
+		.height = view->swayc->current.view_height,
+		.rotation = 0, // TODO
+	};
+
+	view_for_each_popup(view, output_for_each_surface_iterator, &data);
+}
+
 void output_layer_for_each_surface(struct sway_output *output,
 		struct wl_list *layer_surfaces, sway_surface_iterator_func_t iterator,
 		void *user_data) {
@@ -295,8 +312,9 @@ static void send_frame_done_container_iterator(struct sway_container *con,
 		return;
 	}
 
-	output_view_for_each_surface(data->output, con->sway_view,
+	// Toplevels only
-		send_frame_done_iterator, data->when);
+	output_surface_for_each_surface(data->output, con->sway_view->surface,
+		con->x, con->y, send_frame_done_iterator, data->when);
 }
 
 static void send_frame_done_container(struct sway_output *output,
@@ -309,6 +327,27 @@ static void send_frame_done_container(struct sway_output *output,
 		send_frame_done_container_iterator, &data);
 }
 
+static void send_frame_done_popup_iterator(struct sway_output *output,
+		struct wlr_surface *surface, struct wlr_box *box, float rotation,
+		void *data) {
+	// Send frame done to this popup's surface
+	send_frame_done_iterator(output, surface, box, rotation, data);
+
+	// Send frame done to this popup's child toplevels
+	output_surface_for_each_surface(output, surface, box->x, box->y,
+			send_frame_done_iterator, data);
+}
+
+static void send_frame_done_popups(struct sway_output *output,
+		struct sway_view *view, struct timespec *when) {
+	struct send_frame_done_data data = {
+		.output = output,
+		.when = when,
+	};
+	output_view_for_each_popup(output, view,
+			send_frame_done_popup_iterator, &data);
+}
+
 static void send_frame_done(struct sway_output *output, struct timespec *when) {
 	if (output_has_opaque_overlay_layer_surface(output)) {
 		goto send_frame_overlay;
@@ -346,6 +385,13 @@ static void send_frame_done(struct sway_output *output, struct timespec *when) {
 			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP], when);
 	}
 
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
+	struct sway_container *focus = seat_get_focus(seat);
+	if (focus && focus->type == C_VIEW) {
+		send_frame_done_popups(output, focus->sway_view, when);
+	}
+
+
 send_frame_overlay:
 	send_frame_done_layer(output,
 		&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY], when);

sway/desktop/render.c

@@ -186,13 +186,36 @@ static void premultiply_alpha(float color[4], float opacity) {
 	color[2] *= color[3];
 }
 
-static void render_view_surfaces(struct sway_view *view,
+static void render_view_toplevels(struct sway_view *view,
 		struct sway_output *output, pixman_region32_t *damage, float alpha) {
 	struct render_data data = {
 		.damage = damage,
 		.alpha = alpha,
 	};
-	output_view_for_each_surface(output, view, render_surface_iterator, &data);
+	// Render all toplevels without descending into popups
+	output_surface_for_each_surface(output, view->surface,
+			view->swayc->current.view_x, view->swayc->current.view_y,
+			render_surface_iterator, &data);
+}
+
+static void render_popup_iterator(struct sway_output *output,
+		struct wlr_surface *surface, struct wlr_box *box, float rotation,
+		void *data) {
+	// Render this popup's surface
+	render_surface_iterator(output, surface, box, rotation, data);
+
+	// Render this popup's child toplevels
+	output_surface_for_each_surface(output, surface, box->x, box->y,
+			render_surface_iterator, data);
+}
+
+static void render_view_popups(struct sway_view *view,
+		struct sway_output *output, pixman_region32_t *damage, float alpha) {
+	struct render_data data = {
+		.damage = damage,
+		.alpha = alpha,
+	};
+	output_view_for_each_popup(output, view, render_popup_iterator, &data);
 }
 
 static void render_saved_view(struct sway_view *view,
@@ -241,7 +264,7 @@ static void render_view(struct sway_output *output, pixman_region32_t *damage,
 	if (view->swayc->instructions->length) {
 		render_saved_view(view, output, damage, view->swayc->alpha);
 	} else {
-		render_view_surfaces(view, output, damage, view->swayc->alpha);
+		render_view_toplevels(view, output, damage, view->swayc->alpha);
 	}
 
 	if (view->using_csd) {
@@ -845,7 +868,7 @@ void output_render(struct sway_output *output, struct timespec *when,
 				render_saved_view(fullscreen_con->sway_view,
 						output, damage, 1.0f);
 			} else {
-				render_view_surfaces(fullscreen_con->sway_view,
+				render_view_toplevels(fullscreen_con->sway_view,
 						output, damage, 1.0f);
 			}
 		} else {
@@ -881,6 +904,12 @@ void output_render(struct sway_output *output, struct timespec *when,
 			&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP]);
 	}
 
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
+	struct sway_container *focus = seat_get_focus(seat);
+	if (focus && focus->type == C_VIEW) {
+		render_view_popups(focus->sway_view, output, damage, focus->alpha);
+	}
+
 render_overlay:
 	render_layer(output, damage,
 		&output->layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY]);

sway/desktop/xdg_shell.c

@@ -179,6 +179,14 @@ static void for_each_surface(struct sway_view *view,
 		user_data);
 }
 
+static void for_each_popup(struct sway_view *view,
+		wlr_surface_iterator_func_t iterator, void *user_data) {
+	if (xdg_shell_view_from_view(view) == NULL) {
+		return;
+	}
+	wlr_xdg_surface_for_each_popup(view->wlr_xdg_surface, iterator, user_data);
+}
+
 static void _close(struct sway_view *view) {
 	if (xdg_shell_view_from_view(view) == NULL) {
 		return;
@@ -207,6 +215,7 @@ static const struct sway_view_impl view_impl = {
 	.set_fullscreen = set_fullscreen,
 	.wants_floating = wants_floating,
 	.for_each_surface = for_each_surface,
+	.for_each_popup = for_each_popup,
 	.close = _close,
 	.destroy = destroy,
 };

sway/desktop/xdg_shell_v6.c

@@ -175,6 +175,15 @@ static void for_each_surface(struct sway_view *view,
 		user_data);
 }
 
+static void for_each_popup(struct sway_view *view,
+		wlr_surface_iterator_func_t iterator, void *user_data) {
+	if (xdg_shell_v6_view_from_view(view) == NULL) {
+		return;
+	}
+	wlr_xdg_surface_v6_for_each_popup(view->wlr_xdg_surface_v6, iterator,
+		user_data);
+}
+
 static void _close(struct sway_view *view) {
 	if (xdg_shell_v6_view_from_view(view) == NULL) {
 		return;
@@ -203,6 +212,7 @@ static const struct sway_view_impl view_impl = {
 	.set_fullscreen = set_fullscreen,
 	.wants_floating = wants_floating,
 	.for_each_surface = for_each_surface,
+	.for_each_popup = for_each_popup,
 	.close = _close,
 	.destroy = destroy,
 };

sway/input/cursor.c

@@ -109,9 +109,6 @@ static struct sway_container *container_at_coords(
 	}
 
 	struct sway_container *c;
-	if ((c = floating_container_at(lx, ly, surface, sx, sy))) {
-		return c;
-	}
 	if ((c = container_at(ws, lx, ly, surface, sx, sy))) {
 		return c;
 	}

sway/tree/container.c

@@ -561,10 +561,15 @@ static struct sway_container *container_at_view(struct sway_container *swayc,
 		*sx = _sx;
 		*sy = _sy;
 		*surface = _surface;
+		return swayc;
 	}
-	return swayc;
+	return NULL;
 }
 
+static struct sway_container *tiling_container_at(
+		struct sway_container *con, double lx, double ly,
+		struct wlr_surface **surface, double *sx, double *sy);
+
 /**
  * container_at for a container with layout L_TABBED.
  */
@@ -591,7 +596,7 @@ static struct sway_container *container_at_tabbed(struct sway_container *parent,
 	// Surfaces
 	struct sway_container *current = seat_get_active_child(seat, parent);
 
-	return container_at(current, lx, ly, surface, sx, sy);
+	return tiling_container_at(current, lx, ly, surface, sx, sy);
 }
 
 /**
@@ -616,7 +621,7 @@ static struct sway_container *container_at_stacked(
 	// Surfaces
 	struct sway_container *current = seat_get_active_child(seat, parent);
 
-	return container_at(current, lx, ly, surface, sx, sy);
+	return tiling_container_at(current, lx, ly, surface, sx, sy);
 }
 
 /**
@@ -634,45 +639,13 @@ static struct sway_container *container_at_linear(struct sway_container *parent,
 			.height = child->height,
 		};
 		if (wlr_box_contains_point(&box, lx, ly)) {
-			return container_at(child, lx, ly, surface, sx, sy);
+			return tiling_container_at(child, lx, ly, surface, sx, sy);
 		}
 	}
 	return NULL;
 }
 
-struct sway_container *container_at(struct sway_container *parent,
+static struct sway_container *floating_container_at(double lx, double ly,
-		double lx, double ly,
-		struct wlr_surface **surface, double *sx, double *sy) {
-	if (!sway_assert(parent->type >= C_WORKSPACE,
-				"Expected workspace or deeper")) {
-		return NULL;
-	}
-	if (parent->type == C_VIEW) {
-		return container_at_view(parent, lx, ly, surface, sx, sy);
-	}
-	if (!parent->children->length) {
-		return NULL;
-	}
-
-	switch (parent->layout) {
-	case L_HORIZ:
-	case L_VERT:
-		return container_at_linear(parent, lx, ly, surface, sx, sy);
-	case L_TABBED:
-		return container_at_tabbed(parent, lx, ly, surface, sx, sy);
-	case L_STACKED:
-		return container_at_stacked(parent, lx, ly, surface, sx, sy);
-	case L_FLOATING:
-		sway_assert(false, "Didn't expect to see floating here");
-		return NULL;
-	case L_NONE:
-		return NULL;
-	}
-
-	return NULL;
-}
-
-struct sway_container *floating_container_at(double lx, double ly,
 		struct wlr_surface **surface, double *sx, double *sy) {
 	for (int i = 0; i < root_container.children->length; ++i) {
 		struct sway_container *output = root_container.children->items[i];
@@ -694,7 +667,8 @@ struct sway_container *floating_container_at(double lx, double ly,
 					.height = floater->height,
 				};
 				if (wlr_box_contains_point(&box, lx, ly)) {
-					return container_at(floater, lx, ly, surface, sx, sy);
+					return tiling_container_at(floater, lx, ly,
+							surface, sx, sy);
 				}
 			}
 		}
@@ -702,6 +676,90 @@ struct sway_container *floating_container_at(double lx, double ly,
 	return NULL;
 }
 
+static struct sway_container *tiling_container_at(
+		struct sway_container *con, double lx, double ly,
+		struct wlr_surface **surface, double *sx, double *sy) {
+	if (con->type == C_VIEW) {
+		return container_at_view(con, lx, ly, surface, sx, sy);
+	}
+	if (!con->children->length) {
+		return NULL;
+	}
+
+	switch (con->layout) {
+	case L_HORIZ:
+	case L_VERT:
+		return container_at_linear(con, lx, ly, surface, sx, sy);
+	case L_TABBED:
+		return container_at_tabbed(con, lx, ly, surface, sx, sy);
+	case L_STACKED:
+		return container_at_stacked(con, lx, ly, surface, sx, sy);
+	case L_FLOATING:
+		sway_assert(false, "Didn't expect to see floating here");
+		return NULL;
+	case L_NONE:
+		return NULL;
+	}
+	return NULL;
+}
+
+static bool surface_is_popup(struct wlr_surface *surface) {
+	if (wlr_surface_is_xdg_surface(surface)) {
+		struct wlr_xdg_surface *xdg_surface =
+			wlr_xdg_surface_from_wlr_surface(surface);
+		while (xdg_surface) {
+			if (xdg_surface->role == WLR_XDG_SURFACE_ROLE_POPUP) {
+				return true;
+			}
+			xdg_surface = xdg_surface->toplevel->parent;
+		}
+		return false;
+	}
+
+	if (wlr_surface_is_xdg_surface_v6(surface)) {
+		struct wlr_xdg_surface_v6 *xdg_surface_v6 =
+			wlr_xdg_surface_v6_from_wlr_surface(surface);
+		while (xdg_surface_v6) {
+			if (xdg_surface_v6->role == WLR_XDG_SURFACE_V6_ROLE_POPUP) {
+				return true;
+			}
+			xdg_surface_v6 = xdg_surface_v6->toplevel->parent;
+		}
+		return false;
+	}
+
+	return false;
+}
+
+struct sway_container *container_at(struct sway_container *workspace,
+		double lx, double ly,
+		struct wlr_surface **surface, double *sx, double *sy) {
+	if (!sway_assert(workspace->type == C_WORKSPACE, "Expected a workspace")) {
+		return NULL;
+	}
+	struct sway_container *c;
+	// Focused view's popups
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
+	struct sway_container *focus =
+		seat_get_focus_inactive(seat, &root_container);
+	if (focus && focus->type == C_VIEW) {
+		container_at_view(focus, lx, ly, surface, sx, sy);
+		if (*surface && surface_is_popup(*surface)) {
+			return focus;
+		}
+		*surface = NULL;
+	}
+	// Floating
+	if ((c = floating_container_at(lx, ly, surface, sx, sy))) {
+		return c;
+	}
+	// Tiling
+	if ((c = tiling_container_at(workspace, lx, ly, surface, sx, sy))) {
+		return c;
+	}
+	return NULL;
+}
+
 void container_for_each_descendant_dfs(struct sway_container *container,
 		void (*f)(struct sway_container *container, void *data),
 		void *data) {

sway/tree/view.c

@@ -332,6 +332,16 @@ void view_for_each_surface(struct sway_view *view,
 	}
 }
 
+void view_for_each_popup(struct sway_view *view,
+		wlr_surface_iterator_func_t iterator, void *user_data) {
+	if (!view->surface) {
+		return;
+	}
+	if (view->impl->for_each_popup) {
+		view->impl->for_each_popup(view, iterator, user_data);
+	}
+}
+
 static void view_subsurface_create(struct sway_view *view,
 	struct wlr_subsurface *subsurface);