input/keyboard: check keyboard group before remove

In sway_keyboard_destroy, only remove the keyboard from a keyboard
group, if it is part of a keyboard group. If the keyboard is not part of
a keyboard group, then there is nothing to remove it from

sway/input/keyboard.c

@@ -918,7 +918,9 @@ void sway_keyboard_destroy(struct sway_keyboard *keyboard) {
 	if (!keyboard) {
 		return;
 	}
-	sway_keyboard_group_remove(keyboard);
+	if (keyboard->seat_device->input_device->wlr_device->keyboard->group) {
+		sway_keyboard_group_remove(keyboard);
+	}
 	if (keyboard->keymap) {
 		xkb_keymap_unref(keyboard->keymap);
 	}