diff --git a/sway/desktop/layer_shell.c b/sway/desktop/layer_shell.c
index b60aa487..2d355b74 100644
--- a/sway/desktop/layer_shell.c
+++ b/sway/desktop/layer_shell.c
@@ -257,6 +257,11 @@ static void unmap(struct sway_layer_surface *sway_layer) {
 	}
 	output_damage_surface(output, sway_layer->geo.x, sway_layer->geo.y,
 		sway_layer->layer_surface->surface, true);
+
+	struct sway_seat *seat = input_manager_current_seat(input_manager);
+	if (seat->focused_layer == sway_layer->layer_surface) {
+		seat_set_focus_layer(seat, NULL);
+	}
 }
 
 static void handle_destroy(struct wl_listener *listener, void *data) {
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 0e539b70..071ef020 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -658,7 +658,8 @@ void seat_set_focus_layer(struct sway_seat *seat,
 		struct wlr_layer_surface *layer) {
 	if (!layer && seat->focused_layer) {
 		seat->focused_layer = NULL;
-		struct sway_container *previous = seat_get_focus(seat);
+		struct sway_container *previous =
+			seat_get_focus_inactive(seat, &root_container);
 		if (previous) {
 			wlr_log(L_DEBUG, "Returning focus to %p %s '%s'", previous,
 					container_type_to_str(previous->type), previous->name);