render: use DRM formats in wlr_renderer_read_pixels

backend/x11/output.c

@@ -427,7 +427,7 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 	wlr_renderer_end(x11->renderer);
 
 	bool result = wlr_renderer_read_pixels(
-		x11->renderer, WL_SHM_FORMAT_ARGB8888, NULL,
+		x11->renderer, DRM_FORMAT_ARGB8888, NULL,
 		width * 4, width, height, 0, 0, 0, 0,
 		data);
 

include/wlr/render/interface.h

@@ -43,7 +43,7 @@ struct wlr_renderer_impl {
 	const struct wlr_drm_format_set *(*get_dmabuf_render_formats)(
 		struct wlr_renderer *renderer);
 	uint32_t (*preferred_read_format)(struct wlr_renderer *renderer);
-	bool (*read_pixels)(struct wlr_renderer *renderer, enum wl_shm_format fmt,
+	bool (*read_pixels)(struct wlr_renderer *renderer, uint32_t fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 		void *data);

include/wlr/render/wlr_renderer.h

@@ -110,7 +110,7 @@ const struct wlr_drm_format_set *wlr_renderer_get_dmabuf_texture_formats(
  * If `flags` is not NULl, the caller indicates that it accepts frame flags
  * defined in `enum wlr_renderer_read_pixels_flags`.
  */
-bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
+bool wlr_renderer_read_pixels(struct wlr_renderer *r, uint32_t fmt,
 	uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 	uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y, void *data);
 

render/gles2/renderer.c

@@ -467,14 +467,14 @@ static uint32_t gles2_preferred_read_format(
 }
 
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
-		enum wl_shm_format wl_fmt, uint32_t *flags, uint32_t stride,
+		uint32_t drm_format, uint32_t *flags, uint32_t stride,
 		uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y,
 		uint32_t dst_x, uint32_t dst_y, void *data) {
 	struct wlr_gles2_renderer *renderer =
 		gles2_get_renderer_in_context(wlr_renderer);
 
 	const struct wlr_gles2_pixel_format *fmt =
-		get_gles2_format_from_drm(convert_wl_shm_format_to_drm(wl_fmt));
+		get_gles2_format_from_drm(drm_format);
 	if (fmt == NULL) {
 		wlr_log(WLR_ERROR, "Cannot read pixels: unsupported pixel format");
 		return false;

render/wlr_renderer.c

@@ -189,7 +189,7 @@ const struct wlr_drm_format_set *wlr_renderer_get_dmabuf_render_formats(
 	return r->impl->get_dmabuf_render_formats(r);
 }
 
-bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
+bool wlr_renderer_read_pixels(struct wlr_renderer *r, uint32_t fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 		void *data) {

types/wlr_screencopy_v1.c

@@ -233,7 +233,8 @@ static void frame_handle_output_precommit(struct wl_listener *listener,
 	int x = frame->box.x;
 	int y = frame->box.y;
 
-	enum wl_shm_format fmt = wl_shm_buffer_get_format(shm_buffer);
+	enum wl_shm_format wl_shm_format = wl_shm_buffer_get_format(shm_buffer);
+	uint32_t drm_format = convert_wl_shm_format_to_drm(wl_shm_format);
 	int32_t width = wl_shm_buffer_get_width(shm_buffer);
 	int32_t height = wl_shm_buffer_get_height(shm_buffer);
 	int32_t stride = wl_shm_buffer_get_stride(shm_buffer);
@@ -241,7 +242,7 @@ static void frame_handle_output_precommit(struct wl_listener *listener,
 	wl_shm_buffer_begin_access(shm_buffer);
 	void *data = wl_shm_buffer_get_data(shm_buffer);
 	uint32_t renderer_flags = 0;
-	bool ok = wlr_renderer_read_pixels(renderer, fmt, &renderer_flags,
+	bool ok = wlr_renderer_read_pixels(renderer, drm_format, &renderer_flags,
 		stride, width, height, x, y, 0, 0, data);
 	uint32_t flags = renderer_flags & WLR_RENDERER_READ_PIXELS_Y_INVERT ?
 		ZWLR_SCREENCOPY_FRAME_V1_FLAGS_Y_INVERT : 0;