render: unify getter for texture formats

Instead of having separate getters for shm formats and DMA-BUF
formats, use the same pattern as wlr_output_impl.get_primary_formats
with a single function which takes buffer caps as input.

include/render/gles2.h

@@ -44,6 +44,8 @@ struct wlr_gles2_renderer {
 	struct wlr_egl *egl;
 	int drm_fd;
 
+	struct wlr_drm_format_set shm_texture_formats;
+
 	const char *exts_str;
 	struct {
 		bool EXT_read_format_bgra;
@@ -143,8 +145,8 @@ bool is_gles2_pixel_format_supported(const struct wlr_gles2_renderer *renderer,
 const struct wlr_gles2_pixel_format *get_gles2_format_from_drm(uint32_t fmt);
 const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
 	GLint gl_format, GLint gl_type, bool alpha);
-const uint32_t *get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
+void get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
-	size_t *len);
+	struct wlr_drm_format_set *out);
 
 GLuint gles2_buffer_get_fbo(struct wlr_gles2_buffer *buffer);
 

include/render/vulkan.h

@@ -60,11 +60,7 @@ struct wlr_vk_device {
 	struct wlr_vk_format_props *format_props;
 	struct wlr_drm_format_set dmabuf_render_formats;
 	struct wlr_drm_format_set dmabuf_texture_formats;
-
+	struct wlr_drm_format_set shm_texture_formats;
-	// supported formats for textures (contains only those formats
-	// that support everything we need for textures)
-	uint32_t shm_format_count;
-	uint32_t *shm_formats; // to implement vulkan_get_shm_texture_formats
 };
 
 // Tries to find the VkPhysicalDevice for the given drm fd.

include/wlr/render/interface.h

@@ -20,10 +20,8 @@ struct wlr_box;
 struct wlr_fbox;
 
 struct wlr_renderer_impl {
-	const uint32_t *(*get_shm_texture_formats)(
+	const struct wlr_drm_format_set *(*get_texture_formats)(
-		struct wlr_renderer *renderer, size_t *len);
+		struct wlr_renderer *renderer, uint32_t buffer_caps);
-	const struct wlr_drm_format_set *(*get_dmabuf_texture_formats)(
-		struct wlr_renderer *renderer);
 	const struct wlr_drm_format_set *(*get_render_formats)(
 		struct wlr_renderer *renderer);
 	void (*destroy)(struct wlr_renderer *renderer);

include/wlr/render/wlr_renderer.h

@@ -50,11 +50,15 @@ struct wlr_renderer {
 struct wlr_renderer *wlr_renderer_autocreate(struct wlr_backend *backend);
 
 /**
- * Get the shared-memory formats supporting import usage. Buffers allocated
+ * Get the formats supporting sampling usage.
- * with a format from this list may be imported via wlr_texture_from_pixels().
+ *
+ * The buffer capabilities must be passed in.
+ *
+ * Buffers allocated with a format from this list may be passed to
+ * wlr_texture_from_buffer().
  */
-const uint32_t *wlr_renderer_get_shm_texture_formats(
+const struct wlr_drm_format_set *wlr_renderer_get_texture_formats(
-	struct wlr_renderer *r, size_t *len);
+	struct wlr_renderer *r, uint32_t buffer_caps);
 /**
  * Get the DMA-BUF formats supporting sampling usage. Buffers allocated with
  * a format from this list may be imported via wlr_texture_from_dmabuf().

render/gles2/pixel_format.c

@@ -150,16 +150,13 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
 	return NULL;
 }
 
-const uint32_t *get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
+void get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
-		size_t *len) {
+		struct wlr_drm_format_set *out) {
-	static uint32_t shm_formats[sizeof(formats) / sizeof(formats[0])];
-	size_t j = 0;
 	for (size_t i = 0; i < sizeof(formats) / sizeof(formats[0]); i++) {
 		if (!is_gles2_pixel_format_supported(renderer, &formats[i])) {
 			continue;
 		}
-		shm_formats[j++] = formats[i].drm_format;
+		wlr_drm_format_set_add(out, formats[i].drm_format, DRM_FORMAT_MOD_INVALID);
+		wlr_drm_format_set_add(out, formats[i].drm_format, DRM_FORMAT_MOD_LINEAR);
 	}
-	*len = j;
-	return shm_formats;
 }

render/gles2/renderer.c

@@ -165,16 +165,16 @@ error_buffer:
 	return NULL;
 }
 
-static const uint32_t *gles2_get_shm_texture_formats(
+static const struct wlr_drm_format_set *gles2_get_texture_formats(
-		struct wlr_renderer *wlr_renderer, size_t *len) {
+		struct wlr_renderer *wlr_renderer, uint32_t buffer_caps) {
-	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
-	return get_gles2_shm_formats(renderer, len);
-}
-
-static const struct wlr_drm_format_set *gles2_get_dmabuf_texture_formats(
-		struct wlr_renderer *wlr_renderer) {
 	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
+	if (buffer_caps & WLR_BUFFER_CAP_DMABUF) {
 		return wlr_egl_get_dmabuf_texture_formats(renderer->egl);
+	} else if (buffer_caps & WLR_BUFFER_CAP_DATA_PTR) {
+		return &renderer->shm_texture_formats;
+	} else {
+		return NULL;
+	}
 }
 
 static const struct wlr_drm_format_set *gles2_get_render_formats(
@@ -234,6 +234,8 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	wlr_egl_unset_current(renderer->egl);
 	wlr_egl_destroy(renderer->egl);
 
+	wlr_drm_format_set_finish(&renderer->shm_texture_formats);
+
 	if (renderer->drm_fd >= 0) {
 		close(renderer->drm_fd);
 	}
@@ -356,8 +358,7 @@ static void gles2_render_timer_destroy(struct wlr_render_timer *wlr_timer) {
 
 static const struct wlr_renderer_impl renderer_impl = {
 	.destroy = gles2_destroy,
-	.get_shm_texture_formats = gles2_get_shm_texture_formats,
+	.get_texture_formats = gles2_get_texture_formats,
-	.get_dmabuf_texture_formats = gles2_get_dmabuf_texture_formats,
 	.get_render_formats = gles2_get_render_formats,
 	.get_drm_fd = gles2_get_drm_fd,
 	.get_render_buffer_caps = gles2_get_render_buffer_caps,
@@ -684,6 +685,8 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 
 	wlr_egl_unset_current(renderer->egl);
 
+	get_gles2_shm_formats(renderer, &renderer->shm_texture_formats);
+
 	return &renderer->wlr_renderer;
 
 error:

render/pixman/renderer.c

@@ -193,9 +193,14 @@ error_buffer:
 	return NULL;
 }
 
-static const uint32_t *pixman_get_shm_texture_formats(
+static const struct wlr_drm_format_set *pixman_get_texture_formats(
-		struct wlr_renderer *wlr_renderer, size_t *len) {
+		struct wlr_renderer *wlr_renderer, uint32_t buffer_caps) {
-	return get_pixman_drm_formats(len);
+	struct wlr_pixman_renderer *renderer = get_renderer(wlr_renderer);
+	if (buffer_caps & WLR_BUFFER_CAP_DATA_PTR) {
+		return &renderer->drm_formats;
+	} else {
+		return NULL;
+	}
 }
 
 static const struct wlr_drm_format_set *pixman_get_render_formats(
@@ -311,7 +316,7 @@ static struct wlr_render_pass *pixman_begin_buffer_pass(struct wlr_renderer *wlr
 }
 
 static const struct wlr_renderer_impl renderer_impl = {
-	.get_shm_texture_formats = pixman_get_shm_texture_formats,
+	.get_texture_formats = pixman_get_texture_formats,
 	.get_render_formats = pixman_get_render_formats,
 	.texture_from_buffer = pixman_texture_from_buffer,
 	.destroy = pixman_destroy,

render/vulkan/pixel_format.c

@@ -536,8 +536,8 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 			props.shm.features = fmtp.formatProperties.optimalTilingFeatures;
 			props.shm.has_mutable_srgb = has_mutable_srgb;
 
-			dev->shm_formats[dev->shm_format_count] = format->drm;
+			wlr_drm_format_set_add(&dev->shm_texture_formats,
-			++dev->shm_format_count;
+				format->drm, DRM_FORMAT_MOD_LINEAR);
 
 			add_fmt_props = true;
 		}

render/vulkan/renderer.c

@@ -956,17 +956,16 @@ bool vulkan_sync_render_buffer(struct wlr_vk_renderer *renderer,
 	return true;
 }
 
-static const uint32_t *vulkan_get_shm_texture_formats(
+static const struct wlr_drm_format_set *vulkan_get_texture_formats(
-		struct wlr_renderer *wlr_renderer, size_t *len) {
+		struct wlr_renderer *wlr_renderer, uint32_t buffer_caps) {
-	struct wlr_vk_renderer *renderer = vulkan_get_renderer(wlr_renderer);
-	*len = renderer->dev->shm_format_count;
-	return renderer->dev->shm_formats;
-}
-
-static const struct wlr_drm_format_set *vulkan_get_dmabuf_texture_formats(
-		struct wlr_renderer *wlr_renderer) {
 	struct wlr_vk_renderer *renderer = vulkan_get_renderer(wlr_renderer);
+	if (buffer_caps & WLR_BUFFER_CAP_DMABUF) {
 		return &renderer->dev->dmabuf_texture_formats;
+	} else if (buffer_caps & WLR_BUFFER_CAP_DATA_PTR) {
+		return &renderer->dev->shm_texture_formats;
+	} else {
+		return NULL;
+	}
 }
 
 static const struct wlr_drm_format_set *vulkan_get_render_formats(
@@ -1324,8 +1323,7 @@ static struct wlr_render_pass *vulkan_begin_buffer_pass(struct wlr_renderer *wlr
 }
 
 static const struct wlr_renderer_impl renderer_impl = {
-	.get_shm_texture_formats = vulkan_get_shm_texture_formats,
+	.get_texture_formats = vulkan_get_texture_formats,
-	.get_dmabuf_texture_formats = vulkan_get_dmabuf_texture_formats,
 	.get_render_formats = vulkan_get_render_formats,
 	.destroy = vulkan_destroy,
 	.get_drm_fd = vulkan_get_drm_fd,

render/vulkan/vulkan.c

@@ -623,9 +623,8 @@ struct wlr_vk_device *vulkan_device_create(struct wlr_vk_instance *ini,
 
 	size_t max_fmts;
 	const struct wlr_vk_format *fmts = vulkan_get_format_list(&max_fmts);
-	dev->shm_formats = calloc(max_fmts, sizeof(*dev->shm_formats));
 	dev->format_props = calloc(max_fmts, sizeof(*dev->format_props));
-	if (!dev->shm_formats || !dev->format_props) {
+	if (!dev->format_props) {
 		wlr_log_errno(WLR_ERROR, "allocation failed");
 		goto error;
 	}
@@ -657,12 +656,12 @@ void vulkan_device_destroy(struct wlr_vk_device *dev) {
 
 	wlr_drm_format_set_finish(&dev->dmabuf_render_formats);
 	wlr_drm_format_set_finish(&dev->dmabuf_texture_formats);
+	wlr_drm_format_set_finish(&dev->shm_texture_formats);
 
 	for (unsigned i = 0u; i < dev->format_prop_count; ++i) {
 		vulkan_format_props_finish(&dev->format_props[i]);
 	}
 
-	free(dev->shm_formats);
 	free(dev->format_props);
 	free(dev);
 }

render/wlr_renderer.c

@@ -32,7 +32,7 @@
 void wlr_renderer_init(struct wlr_renderer *renderer,
 		const struct wlr_renderer_impl *impl) {
 	assert(impl->begin_buffer_pass);
-	assert(impl->get_shm_texture_formats);
+	assert(impl->get_texture_formats);
 	assert(impl->get_render_buffer_caps);
 
 	*renderer = (struct wlr_renderer){
@@ -57,17 +57,14 @@ void wlr_renderer_destroy(struct wlr_renderer *r) {
 	}
 }
 
-const uint32_t *wlr_renderer_get_shm_texture_formats(struct wlr_renderer *r,
+const struct wlr_drm_format_set *wlr_renderer_get_texture_formats(
-		size_t *len) {
+		struct wlr_renderer *r, uint32_t buffer_caps) {
-	return r->impl->get_shm_texture_formats(r, len);
+	return r->impl->get_texture_formats(r, buffer_caps);
 }
 
 const struct wlr_drm_format_set *wlr_renderer_get_dmabuf_texture_formats(
 		struct wlr_renderer *r) {
-	if (!r->impl->get_dmabuf_texture_formats) {
+	return wlr_renderer_get_texture_formats(r, WLR_BUFFER_CAP_DMABUF);
-		return NULL;
-	}
-	return r->impl->get_dmabuf_texture_formats(r);
 }
 
 const struct wlr_drm_format_set *wlr_renderer_get_render_formats(

types/wlr_shm.c

@@ -9,6 +9,7 @@
 #include <unistd.h>
 #include <wayland-server-protocol.h>
 #include <wlr/interfaces/wlr_buffer.h>
+#include <wlr/render/drm_format_set.h>
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_shm.h>
 #include <wlr/util/log.h>
@@ -540,16 +541,27 @@ struct wlr_shm *wlr_shm_create(struct wl_display *display, uint32_t version,
 
 struct wlr_shm *wlr_shm_create_with_renderer(struct wl_display *display,
 		uint32_t version, struct wlr_renderer *renderer) {
-	size_t formats_len;
+	const struct wlr_drm_format_set *format_set =
-	const uint32_t *formats =
+		wlr_renderer_get_texture_formats(renderer, WLR_BUFFER_CAP_DATA_PTR);
-		wlr_renderer_get_shm_texture_formats(renderer, &formats_len);
+	if (format_set == NULL || format_set->len == 0) {
-	if (formats == NULL) {
 		wlr_log(WLR_ERROR, "Failed to initialize wl_shm: "
 			"cannot get renderer formats");
 		return NULL;
 	}
 
-	return wlr_shm_create(display, version, formats, formats_len);
+	size_t formats_len = format_set->len;
+	uint32_t *formats = calloc(formats_len, sizeof(formats[0]));
+	if (formats == NULL) {
+		return NULL;
+	}
+
+	for (size_t i = 0; i < format_set->len; i++) {
+		formats[i] = format_set->formats[i].format;
+	}
+
+	struct wlr_shm *shm = wlr_shm_create(display, version, formats, formats_len);
+	free(formats);
+	return shm;
 }
 
 static bool shm_has_format(struct wlr_shm *shm, uint32_t shm_format) {