When unfloating, return container to previously focused tiled container

This introduces seat_get_focus_inactive_tiling and updates
`focus mode_toggle` to use it instead, because the previous method
wasn't guaranteed to return a tiling view.

include/sway/input/seat.h

@@ -124,6 +124,9 @@ struct sway_container *seat_get_focus(struct sway_seat *seat);
 struct sway_container *seat_get_focus_inactive(struct sway_seat *seat,
 		struct sway_container *container);
 
+struct sway_container *seat_get_focus_inactive_tiling(struct sway_seat *seat,
+		struct sway_container *container);
+
 /**
  * Descend into the focus stack to find the focus-inactive view. Useful for
  * container placement when they change position in the tree.

sway/commands/focus.c

@@ -35,14 +35,16 @@ static struct cmd_results *focus_mode(struct sway_container *con,
 		struct sway_seat *seat, bool floating) {
 	struct sway_container *ws = con->type == C_WORKSPACE ?
 		con : container_parent(con, C_WORKSPACE);
-	struct sway_container *new_focus = ws;
+	struct sway_container *new_focus = NULL;
 	if (floating) {
-		new_focus = ws->sway_workspace->floating;
-		if (new_focus->children->length == 0) {
-			return cmd_results_new(CMD_SUCCESS, NULL, NULL);
+		new_focus = seat_get_focus_inactive(seat, ws->sway_workspace->floating);
+	} else {
+		new_focus = seat_get_focus_inactive_tiling(seat, ws);
 	}
+	if (!new_focus) {
+		new_focus = ws;
 	}
-	seat_set_focus(seat, seat_get_active_child(seat, new_focus));
+	seat_set_focus(seat, new_focus);
 	return cmd_results_new(CMD_SUCCESS, NULL, NULL);
 }
 

sway/input/seat.c

@@ -124,12 +124,14 @@ static void seat_send_focus(struct sway_container *con,
 }
 
 static struct sway_container *seat_get_focus_by_type(struct sway_seat *seat,
-		struct sway_container *container, enum sway_container_type type) {
+		struct sway_container *container, enum sway_container_type type,
+		bool only_tiling) {
 	if (container->type == C_VIEW) {
 		return container;
 	}
 
-	struct sway_container *floating = container->type == C_WORKSPACE ?
+	struct sway_container *floating =
+		container->type == C_WORKSPACE && !only_tiling ?
 		container->sway_workspace->floating : NULL;
 	if (container->children->length == 0 &&
 			(!floating || floating->children->length == 0)) {
@@ -143,6 +145,10 @@ static struct sway_container *seat_get_focus_by_type(struct sway_seat *seat,
 		}
 
 		if (container_has_child(container, current->container)) {
+			if (only_tiling &&
+					container_is_floating_or_child(current->container)) {
+				continue;
+			}
 			return current->container;
 		}
 		if (floating && container_has_child(floating, current->container)) {
@@ -169,7 +175,7 @@ void seat_focus_inactive_children_for_each(struct sway_seat *seat,
 
 struct sway_container *seat_get_focus_inactive_view(struct sway_seat *seat,
 		struct sway_container *container) {
-	return seat_get_focus_by_type(seat, container, C_VIEW);
+	return seat_get_focus_by_type(seat, container, C_VIEW, false);
 }
 
 static void handle_seat_container_destroy(struct wl_listener *listener,
@@ -191,7 +197,7 @@ static void handle_seat_container_destroy(struct wl_listener *listener,
 	if (set_focus) {
 		struct sway_container *next_focus = NULL;
 		while (next_focus == NULL) {
-			next_focus = seat_get_focus_by_type(seat, parent, C_VIEW);
+			next_focus = seat_get_focus_by_type(seat, parent, C_VIEW, false);
 
 			if (next_focus == NULL && parent->type == C_WORKSPACE) {
 				next_focus = parent;
@@ -648,7 +654,7 @@ void seat_set_focus_warp(struct sway_seat *seat,
 	struct sway_container *new_output_last_ws = NULL;
 	if (last_output && new_output && last_output != new_output) {
 		new_output_last_ws =
-			seat_get_focus_by_type(seat, new_output, C_WORKSPACE);
+			seat_get_focus_by_type(seat, new_output, C_WORKSPACE, false);
 	}
 
 	if (container && container->parent) {
@@ -853,7 +859,12 @@ void seat_set_exclusive_client(struct sway_seat *seat,
 
 struct sway_container *seat_get_focus_inactive(struct sway_seat *seat,
 		struct sway_container *container) {
-	return seat_get_focus_by_type(seat, container, C_TYPES);
+	return seat_get_focus_by_type(seat, container, C_TYPES, false);
+}
+
+struct sway_container *seat_get_focus_inactive_tiling(struct sway_seat *seat,
+		struct sway_container *container) {
+	return seat_get_focus_by_type(seat, container, C_TYPES, true);
 }
 
 struct sway_container *seat_get_active_child(struct sway_seat *seat,

sway/tree/container.c

@@ -1015,6 +1015,7 @@ void container_set_floating(struct sway_container *container, bool enable) {
 		return;
 	}
 
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
 	struct sway_container *workspace = container_parent(container, C_WORKSPACE);
 
 	if (enable) {
@@ -1029,8 +1030,10 @@ void container_set_floating(struct sway_container *container, bool enable) {
 		if (container->scratchpad) {
 			scratchpad_remove_container(container);
 		}
+		struct sway_container *sibling =
+			seat_get_focus_inactive_tiling(seat, workspace);
 		container_remove_child(container);
-		container_add_child(workspace, container);
+		container_add_child(sibling, container);
 		container->width = container->parent->width;
 		container->height = container->parent->height;
 		if (container->type == C_VIEW) {