xwayland: disconnect display destroy listener even if xwayland didn't initialize

xwayland/server.c

@@ -150,7 +150,13 @@ static void server_finish_process(struct wlr_xwayland_server *server) {
 }
 
 static void server_finish_display(struct wlr_xwayland_server *server) {
-	if (!server || server->display == -1) {
+	if (!server) {
+		return;
+	}
+
+	wl_list_remove(&server->display_destroy.link);
+
+	if (server->display == -1) {
 		return;
 	}
 
@@ -158,8 +164,6 @@ static void server_finish_display(struct wlr_xwayland_server *server) {
 	safe_close(server->x_fd[1]);
 	server->x_fd[0] = server->x_fd[1] = -1;
 
-	wl_list_remove(&server->display_destroy.link);
-
 	unlink_display_sockets(server->display);
 	server->display = -1;
 	server->display_name[0] = '\0';