diff --git a/include/rootston/input.h b/include/rootston/input.h
index 5cd7509e..20b73c8a 100644
--- a/include/rootston/input.h
+++ b/include/rootston/input.h
@@ -68,6 +68,7 @@ struct roots_input_event {
 struct roots_drag_icon {
 	struct wlr_surface *surface;
 	struct wl_list link; // roots_input::drag_icons
+	bool mapped;
 
 	int32_t sx;
 	int32_t sy;
diff --git a/include/wlr/types/wlr_data_device.h b/include/wlr/types/wlr_data_device.h
index 70d6dce3..691e2df8 100644
--- a/include/wlr/types/wlr_data_device.h
+++ b/include/wlr/types/wlr_data_device.h
@@ -61,6 +61,8 @@ struct wlr_drag {
 	struct wlr_surface *focus;
 	struct wlr_data_source *source;
 
+	bool cancelling;
+
 	struct wl_listener icon_destroy;
 	struct wl_listener source_destroy;
 	struct wl_listener seat_client_unbound;
diff --git a/rootston/cursor.c b/rootston/cursor.c
index 7307b6b2..31001a9f 100644
--- a/rootston/cursor.c
+++ b/rootston/cursor.c
@@ -405,9 +405,6 @@ static void handle_drag_icon_destroy(struct wl_listener *listener, void *data) {
 static void handle_drag_icon_commit(struct wl_listener *listener, void *data) {
 	struct roots_drag_icon *drag_icon =
 		wl_container_of(listener, drag_icon, surface_commit);
-	// TODO the spec hints at rules that can determine whether the drag icon is
-	// mapped here, but it is not completely clear so we need to test more
-	// toolkits to see how we should interpret the surface state here.
 	drag_icon->sx += drag_icon->surface->current->sx;
 	drag_icon->sy += drag_icon->surface->current->sy;
 }
@@ -431,6 +428,7 @@ static void handle_pointer_grab_begin(struct wl_listener *listener,
 
 			struct roots_drag_icon *drag_icon =
 				calloc(1, sizeof(struct roots_drag_icon));
+			drag_icon->mapped = true;
 			drag_icon->surface = drag->icon;
 			wl_list_insert(&input->drag_icons, &drag_icon->link);
 
@@ -448,6 +446,18 @@ static void handle_pointer_grab_begin(struct wl_listener *listener,
 static void handle_pointer_grab_end(struct wl_listener *listener, void *data) {
 	struct roots_input *input =
 		wl_container_of(listener, input, pointer_grab_end);
+	struct wlr_seat_pointer_grab *grab = data;
+
+	if (grab->interface == &wlr_data_device_pointer_drag_interface) {
+		struct wlr_drag *drag = grab->data;
+		struct roots_drag_icon *icon;
+		wl_list_for_each(icon, &input->drag_icons, link) {
+			if (icon->surface == drag->icon) {
+				icon->mapped = false;
+			}
+		}
+	}
+
 	cursor_update_position(input, 0);
 }
 
diff --git a/rootston/output.c b/rootston/output.c
index 9e1cd4d5..baa7b6cc 100644
--- a/rootston/output.c
+++ b/rootston/output.c
@@ -152,6 +152,9 @@ static void output_frame_notify(struct wl_listener *listener, void *data) {
 
 	struct roots_drag_icon *drag_icon = NULL;
 	wl_list_for_each(drag_icon, &server->input->drag_icons, link) {
+		if (!drag_icon->mapped) {
+			continue;
+		}
 		struct wlr_surface *icon = drag_icon->surface;
 		struct wlr_cursor *cursor = server->input->cursor;
 		double icon_x = cursor->x + drag_icon->sx;
diff --git a/types/wlr_data_device.c b/types/wlr_data_device.c
index 3914c703..5800c0f3 100644
--- a/types/wlr_data_device.c
+++ b/types/wlr_data_device.c
@@ -432,18 +432,23 @@ static void wlr_drag_set_focus(struct wlr_drag *drag,
 }
 
 static void wlr_drag_end(struct wlr_drag *drag) {
-	if (drag->icon) {
-		wl_list_remove(&drag->icon_destroy.link);
-	}
+	if (!drag->cancelling) {
+		drag->cancelling = true;
+		wlr_seat_pointer_end_grab(drag->pointer_grab.seat);
+		wlr_seat_keyboard_end_grab(drag->keyboard_grab.seat);
 
-	if (drag->source) {
-		wl_list_remove(&drag->source_destroy.link);
-	}
+		if (drag->source) {
+			wl_list_remove(&drag->source_destroy.link);
+		}
+
+		wlr_drag_set_focus(drag, NULL, 0, 0);
 
-	wlr_drag_set_focus(drag, NULL, 0, 0);
-	wlr_seat_pointer_end_grab(drag->pointer_grab.seat);
-	wlr_seat_keyboard_end_grab(drag->keyboard_grab.seat);
-	free(drag);
+		if (drag->icon) {
+			wl_list_remove(&drag->icon_destroy.link);
+		}
+
+		free(drag);
+	}
 }
 
 static void pointer_drag_enter(struct wlr_seat_pointer_grab *grab,
@@ -588,8 +593,6 @@ static bool seat_client_start_drag(struct wlr_seat_client *client,
 	wlr_seat_keyboard_start_grab(seat, &drag->keyboard_grab);
 	wlr_seat_pointer_start_grab(seat, &drag->pointer_grab);
 
-	// TODO keyboard grab
-
 	return true;
 }
 
diff --git a/types/wlr_seat.c b/types/wlr_seat.c
index 1563c300..0b3cbd0f 100644
--- a/types/wlr_seat.c
+++ b/types/wlr_seat.c
@@ -547,6 +547,9 @@ void wlr_seat_pointer_end_grab(struct wlr_seat *wlr_seat) {
 	if (grab != wlr_seat->pointer_state.default_grab) {
 		wlr_seat->pointer_state.grab = wlr_seat->pointer_state.default_grab;
 		wl_signal_emit(&wlr_seat->events.pointer_grab_end, grab);
+		if (grab->interface->cancel) {
+			grab->interface->cancel(grab);
+		}
 	}
 }
 
@@ -665,6 +668,9 @@ void wlr_seat_keyboard_end_grab(struct wlr_seat *wlr_seat) {
 	if (grab != wlr_seat->keyboard_state.default_grab) {
 		wlr_seat->keyboard_state.grab = wlr_seat->keyboard_state.default_grab;
 		wl_signal_emit(&wlr_seat->events.keyboard_grab_end, grab);
+		if (grab->interface->cancel) {
+			grab->interface->cancel(grab);
+		}
 	}
 }