tree/view: fix smart gaps when ancestor container is tabbed or stacked

Fixes #5406.

include/sway/tree/view.h

@@ -239,11 +239,12 @@ uint32_t view_configure(struct sway_view *view, double lx, double ly, int width,
 bool view_inhibit_idle(struct sway_view *view);
 
 /**
- * Whether or not the view is the only visible view in its tree. If the view
- * is tiling, there may be floating views. If the view is floating, there may
- * be tiling views or views in a different floating container.
+ * Whether or not this view's most distant ancestor (possibly itself) is the
+ * only visible node in its tree. If the view is tiling, there may be floating
+ * views. If the view is floating, there may be tiling views or views in a
+ * different floating container.
  */
-bool view_is_only_visible(struct sway_view *view);
+bool view_ancestor_is_only_visible(struct sway_view *view);
 
 /**
  * Configure the view's position and size based on the container's position and

sway/tree/view.c

@@ -192,21 +192,22 @@ bool view_inhibit_idle(struct sway_view *view) {
 		|| sway_idle_inhibit_v1_is_active(application_inhibitor);
 }
 
-bool view_is_only_visible(struct sway_view *view) {
-	bool only_view = true;
+bool view_ancestor_is_only_visible(struct sway_view *view) {
+	bool only_visible = true;
 	struct sway_container *con = view->container;
 	while (con) {
 		enum sway_container_layout layout = container_parent_layout(con);
 		if (layout != L_TABBED && layout != L_STACKED) {
 			list_t *siblings = container_get_siblings(con);
 			if (siblings && siblings->length > 1) {
-				only_view = false;
-				break;
+				only_visible = false;
 			}
+		} else {
+			only_visible = true;
 		}
 		con = con->parent;
 	}
-	return only_view;
+	return only_visible;
 }
 
 static bool gaps_to_edge(struct sway_view *view) {
@@ -247,7 +248,7 @@ void view_autoconfigure(struct sway_view *view) {
 		bool smart = config->hide_edge_borders_smart == ESMART_ON ||
 			(config->hide_edge_borders_smart == ESMART_NO_GAPS &&
 			!gaps_to_edge(view));
-		bool hide_smart = smart && view_is_only_visible(view);
+		bool hide_smart = smart && view_ancestor_is_only_visible(view);
 
 		if (config->hide_edge_borders == E_BOTH
 				|| config->hide_edge_borders == E_VERTICAL || hide_smart) {

sway/tree/workspace.c

@@ -709,7 +709,7 @@ void workspace_add_gaps(struct sway_workspace *ws) {
 		if (focus && !focus->view) {
 			focus = seat_get_focus_inactive_view(seat, &focus->node);
 		}
-		if (focus && focus->view && view_is_only_visible(focus->view)) {
+		if (focus && focus->view && view_ancestor_is_only_visible(focus->view)) {
 			ws->current_gaps.top = 0;
 			ws->current_gaps.right = 0;
 			ws->current_gaps.bottom = 0;