diff --git a/swaylock/password.c b/swaylock/password.c
index 3059203a..3bd113ad 100644
--- a/swaylock/password.c
+++ b/swaylock/password.c
@@ -146,14 +146,6 @@ void swaylock_handle_key(struct swaylock_state *state,
 		schedule_indicator_clear(state);
 		break;
 	case XKB_KEY_Caps_Lock:
-		/* The state is getting active after this
-		 * so we need to manually toggle it */
-		state->xkb.caps_lock = !state->xkb.caps_lock;
-		state->auth_state = AUTH_STATE_INPUT_NOP;
-		damage_state(state);
-		schedule_indicator_clear(state);
-		schedule_password_clear(state);
-		break;
 	case XKB_KEY_Shift_L:
 	case XKB_KEY_Shift_R:
 	case XKB_KEY_Control_L:
diff --git a/swaylock/seat.c b/swaylock/seat.c
index 7b72114f..f0b1385e 100644
--- a/swaylock/seat.c
+++ b/swaylock/seat.c
@@ -63,8 +63,12 @@ static void keyboard_modifiers(void *data, struct wl_keyboard *wl_keyboard,
 	struct swaylock_state *state = data;
 	xkb_state_update_mask(state->xkb.state,
 		mods_depressed, mods_latched, mods_locked, 0, 0, group);
-	state->xkb.caps_lock = xkb_state_mod_name_is_active(state->xkb.state,
+	int caps_lock = xkb_state_mod_name_is_active(state->xkb.state,
 		XKB_MOD_NAME_CAPS, XKB_STATE_MODS_LOCKED);
+	if (caps_lock != state->xkb.caps_lock) {
+		state->xkb.caps_lock = caps_lock;
+		damage_state(state);
+	}
 	state->xkb.control = xkb_state_mod_name_is_active(state->xkb.state,
 		XKB_MOD_NAME_CTRL,
 		XKB_STATE_MODS_DEPRESSED | XKB_STATE_MODS_LATCHED);