diff --git a/include/util/set.h b/include/util/set.h
index 41099621..d6330108 100644
--- a/include/util/set.h
+++ b/include/util/set.h
@@ -2,24 +2,28 @@
 #define UTIL_SET_H
 
 #include <stdint.h>
-#include <stdlib.h>
 #include <stdbool.h>
-
-size_t push_zeroes_to_end(uint32_t arr[], size_t n);
+#include <sys/types.h>
 
 /**
- * Add `target` to `values` if it doesn't exist
- * "set"s should only be modified with set_* functions
- * Values MUST be greater than 0
+ * Add target to values.
+ *
+ * Target is added to the end of the set.
+ *
+ * Returns the index of target, or -1 if the set is full or target already
+ * exists.
  */
-bool set_add(uint32_t values[], size_t *len, size_t cap, uint32_t target);
+ssize_t set_add(uint32_t values[], size_t *len, size_t cap, uint32_t target);
 
 /**
- * Remove `target` from `values` if it exists
- * "set"s should only be modified with set_* functions
- * Values MUST be greater than 0
+ * Remove target from values.
+ *
+ * When target is removed, the last element of the set is moved to where
+ * target was.
+ *
+ * Returns the previous index of target, or -1 if target wasn't in values.
  */
-bool set_remove(uint32_t values[], size_t *len, size_t cap, uint32_t target);
+ssize_t set_remove(uint32_t values[], size_t *len, size_t cap, uint32_t target);
 
 #endif
 
diff --git a/types/tablet_v2/wlr_tablet_v2_tool.c b/types/tablet_v2/wlr_tablet_v2_tool.c
index 9590cd88..df8ab761 100644
--- a/types/tablet_v2/wlr_tablet_v2_tool.c
+++ b/types/tablet_v2/wlr_tablet_v2_tool.c
@@ -257,50 +257,26 @@ struct wlr_tablet_tool_client_v2 *tablet_tool_client_from_resource(struct wl_res
 	return wl_resource_get_user_data(resource);
 }
 
-
-/* Actual protocol foo */
-
-// Button 0 is KEY_RESERVED in input-event-codes on linux (and freebsd)
 static ssize_t tablet_tool_button_update(struct wlr_tablet_v2_tablet_tool *tool,
 		uint32_t button, enum zwp_tablet_pad_v2_button_state state) {
-	bool found = false;
-	size_t i = 0;
-	for (; i < tool->num_buttons; ++i) {
-		if (tool->pressed_buttons[i] == button) {
-			found = true;
-			wlr_log(WLR_DEBUG, "Found the button \\o/: %u", button);
-			break;
-
-		}
-	}
-
-	if (state == ZWP_TABLET_PAD_V2_BUTTON_STATE_PRESSED && found) {
-		/* Already have the button saved, durr */
-		return -1;
-	}
-
-	if (state == ZWP_TABLET_PAD_V2_BUTTON_STATE_PRESSED && !found) {
-		if (tool->num_buttons < WLR_TABLET_V2_TOOL_BUTTONS_CAP) {
-			i = tool->num_buttons++;
-			tool->pressed_buttons[i] = button;
+	ssize_t i;
+	if (state == ZWP_TABLET_PAD_V2_BUTTON_STATE_PRESSED) {
+		i = set_add(tool->pressed_buttons, &tool->num_buttons,
+			WLR_TABLET_V2_TOOL_BUTTONS_CAP, button);
+		if (i != -1) {
 			tool->pressed_serials[i] = -1;
 		} else {
-			i = -1;
-			wlr_log(WLR_ERROR, "You pressed more than %d tablet tool buttons. "
-				"This is currently not supported by wlroots. Please report this "
-				"with a description of your tablet, since this is either a "
-				"bug, or fancy hardware", WLR_TABLET_V2_TOOL_BUTTONS_CAP);
+			wlr_log(WLR_ERROR, "Failed to add tablet tool button %x", button);
+		}
+	} else {
+		i = set_remove(tool->pressed_buttons, &tool->num_buttons,
+			WLR_TABLET_V2_TOOL_BUTTONS_CAP, button);
+		if (i != -1) {
+			tool->pressed_serials[i] = tool->pressed_serials[tool->num_buttons];
+		} else {
+			wlr_log(WLR_ERROR, "Failed to remove tablet tool button %x", button);
 		}
 	}
-	if (state == ZWP_TABLET_PAD_V2_BUTTON_STATE_RELEASED && found) {
-		wlr_log(WLR_DEBUG, "Removed the button \\o/: %u", button);
-		tool->pressed_buttons[i] = 0;
-		tool->pressed_serials[i] = 0;
-		tool->num_buttons = push_zeroes_to_end(tool->pressed_buttons, WLR_TABLET_V2_TOOL_BUTTONS_CAP);
-		tool->num_buttons = push_zeroes_to_end(tool->pressed_serials, WLR_TABLET_V2_TOOL_BUTTONS_CAP);
-	}
-
-	assert(tool->num_buttons <= WLR_TABLET_V2_TOOL_BUTTONS_CAP);
 	return i;
 }
 
diff --git a/util/set.c b/util/set.c
index b3aa18ff..1366d04f 100644
--- a/util/set.c
+++ b/util/set.c
@@ -1,47 +1,25 @@
 #include "util/set.h"
 
-// https://www.geeksforgeeks.org/move-zeroes-end-array/
-size_t push_zeroes_to_end(uint32_t arr[], size_t n) {
-	size_t count = 0;
-
-	for (size_t i = 0; i < n; i++) {
-		if (arr[i] != 0) {
-			arr[count++] = arr[i];
-		}
-	}
-
-	size_t ret = count;
-
-	while (count < n) {
-		arr[count++] = 0;
-	}
-
-	return ret;
-}
-
-bool set_add(uint32_t values[], size_t *len, size_t cap, uint32_t target) {
-	if (*len == cap) {
-		return false;
-	}
+ssize_t set_add(uint32_t values[], size_t *len, size_t cap, uint32_t target) {
 	for (uint32_t i = 0; i < *len; ++i) {
 		if (values[i] == target) {
-			return false;
+			return i;
 		}
 	}
-	values[(*len)++] = target;
-	return false;
+	if (*len == cap) {
+		return -1;
+	}
+	values[*len] = target;
+	return (*len)++;
 }
 
-bool set_remove(uint32_t values[], size_t *len, size_t cap, uint32_t target) {
+ssize_t set_remove(uint32_t values[], size_t *len, size_t cap, uint32_t target) {
 	for (uint32_t i = 0; i < *len; ++i) {
 		if (values[i] == target) {
-			// Set to 0 and swap with the end element so that
-			// zeroes exist only after all the values.
-			size_t last_elem_pos = --(*len);
-			values[i] = values[last_elem_pos];
-			values[last_elem_pos] = 0;
-			return true;
+			--(*len);
+			values[i] = values[*len];
+			return i;
 		}
 	}
-	return false;
+	return -1;
 }