viewporter: fix src buffer bounds check

The surface scale and transform are applied before the viewport.

Closes: https://gitlab.freedesktop.org/wlroots/wlroots/-/issues/3766

types/wlr_viewporter.c

@@ -3,6 +3,7 @@
 #include <wlr/types/wlr_compositor.h>
 #include <wlr/types/wlr_viewporter.h>
 #include <wlr/util/log.h>
+#include <wlr/util/transform.h>
 #include "viewporter-protocol.h"
 
 #define VIEWPORTER_VERSION 1
@@ -131,6 +132,15 @@ static void viewport_handle_resource_destroy(struct wl_resource *resource) {
 	viewport_destroy(viewport);
 }
 
+static bool check_src_buffer_bounds(const struct wlr_surface_state *state) {
+	int width = state->buffer_width / state->scale;
+	int height = state->buffer_height / state->scale;
+	wlr_output_transform_coords(state->transform, &width, &height);
+
+	struct wlr_fbox box = state->viewport.src;
+	return box.x + box.width <= width && box.y + box.height <= height;
+}
+
 static void viewport_handle_surface_client_commit(struct wl_listener *listener,
 		void *data) {
 	struct wlr_viewport *viewport =
@@ -148,10 +158,7 @@ static void viewport_handle_surface_client_commit(struct wl_listener *listener,
 	}
 
 	if (state->viewport.has_src && state->buffer != NULL &&
-			(state->viewport.src.x + state->viewport.src.width >
-				state->buffer_width ||
-			state->viewport.src.y + state->viewport.src.height >
-				state->buffer_height)) {
+			!check_src_buffer_bounds(state)) {
 		wl_resource_post_error(viewport->resource, WP_VIEWPORT_ERROR_OUT_OF_BUFFER,
 			"source rectangle out of buffer bounds");
 		return;