diff --git a/include/sway/input/seat.h b/include/sway/input/seat.h
index cd36ef5a..ab25788f 100644
--- a/include/sway/input/seat.h
+++ b/include/sway/input/seat.h
@@ -68,6 +68,9 @@ struct sway_seat {
 	double op_ref_width, op_ref_height;  // container's size at start of op
 	double op_ref_con_lx, op_ref_con_ly; // container's x/y at start of op
 
+	uint32_t last_button;
+	uint32_t last_button_serial;
+
 	struct wl_listener focus_destroy;
 	struct wl_listener new_container;
 	struct wl_listener new_drag_icon;
@@ -150,11 +153,15 @@ bool seat_is_input_allowed(struct sway_seat *seat, struct wlr_surface *surface);
 
 void drag_icon_update_position(struct sway_drag_icon *icon);
 
-void seat_begin_move(struct sway_seat *seat, struct sway_container *con);
+void seat_begin_move(struct sway_seat *seat, struct sway_container *con,
+		uint32_t button);
 
 void seat_begin_resize(struct sway_seat *seat, struct sway_container *con,
 		uint32_t button, enum wlr_edges edge);
 
 void seat_end_mouse_operation(struct sway_seat *seat);
 
+void seat_pointer_notify_button(struct sway_seat *seat, uint32_t time_msec,
+		uint32_t button, enum wlr_button_state state);
+
 #endif
diff --git a/sway/desktop/xdg_shell.c b/sway/desktop/xdg_shell.c
index d6c3a9a7..c5d53d1d 100644
--- a/sway/desktop/xdg_shell.c
+++ b/sway/desktop/xdg_shell.c
@@ -1,9 +1,4 @@
 #define _POSIX_C_SOURCE 199309L
-#ifdef __linux__
-#include <linux/input-event-codes.h>
-#elif __FreeBSD__
-#include <dev/evdev/input-event-codes.h>
-#endif
 #include <stdbool.h>
 #include <stdlib.h>
 #include <wayland-server.h>
@@ -259,7 +254,9 @@ static void handle_request_move(struct wl_listener *listener, void *data) {
 	struct sway_view *view = &xdg_shell_view->view;
 	struct wlr_xdg_toplevel_move_event *e = data;
 	struct sway_seat *seat = e->seat->seat->data;
-	seat_begin_move(seat, view->swayc);
+	if (e->serial == seat->last_button_serial) {
+		seat_begin_move(seat, view->swayc, seat->last_button);
+	}
 }
 
 static void handle_request_resize(struct wl_listener *listener, void *data) {
@@ -268,7 +265,9 @@ static void handle_request_resize(struct wl_listener *listener, void *data) {
 	struct sway_view *view = &xdg_shell_view->view;
 	struct wlr_xdg_toplevel_resize_event *e = data;
 	struct sway_seat *seat = e->seat->seat->data;
-	seat_begin_resize(seat, view->swayc, BTN_LEFT, e->edges);
+	if (e->serial == seat->last_button_serial) {
+		seat_begin_resize(seat, view->swayc, seat->last_button, e->edges);
+	}
 }
 
 static void handle_unmap(struct wl_listener *listener, void *data) {
diff --git a/sway/desktop/xdg_shell_v6.c b/sway/desktop/xdg_shell_v6.c
index 241bd9b0..4bd6af5e 100644
--- a/sway/desktop/xdg_shell_v6.c
+++ b/sway/desktop/xdg_shell_v6.c
@@ -1,9 +1,4 @@
 #define _POSIX_C_SOURCE 199309L
-#ifdef __linux__
-#include <linux/input-event-codes.h>
-#elif __FreeBSD__
-#include <dev/evdev/input-event-codes.h>
-#endif
 #include <stdbool.h>
 #include <stdlib.h>
 #include <wayland-server.h>
@@ -254,7 +249,9 @@ static void handle_request_move(struct wl_listener *listener, void *data) {
 	struct sway_view *view = &xdg_shell_v6_view->view;
 	struct wlr_xdg_toplevel_v6_move_event *e = data;
 	struct sway_seat *seat = e->seat->seat->data;
-	seat_begin_move(seat, view->swayc);
+	if (e->serial == seat->last_button_serial) {
+		seat_begin_move(seat, view->swayc, seat->last_button);
+	}
 }
 
 static void handle_request_resize(struct wl_listener *listener, void *data) {
@@ -263,7 +260,9 @@ static void handle_request_resize(struct wl_listener *listener, void *data) {
 	struct sway_view *view = &xdg_shell_v6_view->view;
 	struct wlr_xdg_toplevel_v6_resize_event *e = data;
 	struct sway_seat *seat = e->seat->seat->data;
-	seat_begin_resize(seat, view->swayc, BTN_LEFT, e->edges);
+	if (e->serial == seat->last_button_serial) {
+		seat_begin_resize(seat, view->swayc, seat->last_button, e->edges);
+	}
 }
 
 static void handle_unmap(struct wl_listener *listener, void *data) {
diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index 43721d28..ad0ceb94 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -429,7 +429,7 @@ static void dispatch_cursor_button_floating(struct sway_cursor *cursor,
 
 	// Deny moving or resizing a fullscreen view
 	if (cont->type == C_VIEW && cont->sway_view->is_fullscreen) {
-		wlr_seat_pointer_notify_button(seat->wlr_seat, time_msec, button, state);
+		seat_pointer_notify_button(seat, time_msec, button, state);
 		return;
 	}
 
@@ -442,7 +442,7 @@ static void dispatch_cursor_button_floating(struct sway_cursor *cursor,
 	// Check for beginning move
 	if (button == BTN_LEFT && state == WLR_BUTTON_PRESSED &&
 			(mod_pressed || over_title)) {
-		seat_begin_move(seat, cont);
+		seat_begin_move(seat, cont, BTN_LEFT);
 		return;
 	}
 
@@ -456,7 +456,7 @@ static void dispatch_cursor_button_floating(struct sway_cursor *cursor,
 	}
 
 	// Send event to surface
-	wlr_seat_pointer_notify_button(seat->wlr_seat, time_msec, button, state);
+	seat_pointer_notify_button(seat, time_msec, button, state);
 }
 
 void dispatch_cursor_button(struct sway_cursor *cursor,
@@ -480,8 +480,7 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 		if (layer->current.keyboard_interactive) {
 			seat_set_focus_layer(cursor->seat, layer);
 		}
-		wlr_seat_pointer_notify_button(cursor->seat->wlr_seat,
-				time_msec, button, state);
+		seat_pointer_notify_button(cursor->seat, time_msec, button, state);
 	} else if (cont && container_is_floating(cont)) {
 		dispatch_cursor_button_floating(cursor, time_msec, button, state,
 				surface, sx, sy, cont);
@@ -501,15 +500,12 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 		if (new_ws != old_ws) {
 			seat_set_focus(cursor->seat, cont);
 		}
-		wlr_seat_pointer_notify_button(cursor->seat->wlr_seat,
-				time_msec, button, state);
+		seat_pointer_notify_button(cursor->seat, time_msec, button, state);
 	} else if (cont) {
 		seat_set_focus(cursor->seat, cont);
-		wlr_seat_pointer_notify_button(cursor->seat->wlr_seat,
-				time_msec, button, state);
+		seat_pointer_notify_button(cursor->seat, time_msec, button, state);
 	} else {
-		wlr_seat_pointer_notify_button(cursor->seat->wlr_seat,
-				time_msec, button, state);
+		seat_pointer_notify_button(cursor->seat, time_msec, button, state);
 	}
 
 	transaction_commit_dirty();
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 3a3350e1..4e803efd 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -901,14 +901,15 @@ struct seat_config *seat_get_config(struct sway_seat *seat) {
 	return NULL;
 }
 
-void seat_begin_move(struct sway_seat *seat, struct sway_container *con) {
+void seat_begin_move(struct sway_seat *seat, struct sway_container *con,
+		uint32_t button) {
 	if (!seat->cursor) {
 		wlr_log(WLR_DEBUG, "Ignoring move request due to no cursor device");
 		return;
 	}
 	seat->operation = OP_MOVE;
 	seat->op_container = con;
-	seat->op_button = BTN_LEFT;
+	seat->op_button = button;
 }
 
 void seat_begin_resize(struct sway_seat *seat, struct sway_container *con,
@@ -951,3 +952,10 @@ void seat_end_mouse_operation(struct sway_seat *seat) {
 	seat->operation = OP_NONE;
 	seat->op_container = NULL;
 }
+
+void seat_pointer_notify_button(struct sway_seat *seat, uint32_t time_msec,
+		uint32_t button, enum wlr_button_state state) {
+	seat->last_button = button;
+	seat->last_button_serial = wlr_seat_pointer_notify_button(seat->wlr_seat,
+			time_msec, button, state);
+}