Add support for linux-dmabuf surface hints

References: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/1376

include/sway/server.h

@@ -41,6 +41,8 @@ struct sway_server {
 	struct wlr_compositor *compositor;
 	struct wl_listener compositor_new_surface;
 
+	struct wlr_linux_dmabuf_v1 *linux_dmabuf_v1;
+
 	struct wlr_data_device_manager *data_device_manager;
 
 	struct sway_input_manager *input;

protocols/meson.build

@@ -15,6 +15,7 @@ protocols = [
 	[wl_protocol_dir, 'unstable/xdg-output/xdg-output-unstable-v1.xml'],
 	[wl_protocol_dir, 'unstable/pointer-constraints/pointer-constraints-unstable-v1.xml'],
 	[wl_protocol_dir, 'unstable/tablet/tablet-unstable-v2.xml'],
+	[wl_protocol_dir, 'unstable/linux-dmabuf/linux-dmabuf-unstable-v1.xml'],
 	['wlr-layer-shell-unstable-v1.xml'],
 	['idle.xml'],
 	['wlr-input-inhibitor-unstable-v1.xml'],

sway/server.c

@@ -13,10 +13,12 @@
 #include <wlr/types/wlr_compositor.h>
 #include <wlr/types/wlr_data_control_v1.h>
 #include <wlr/types/wlr_drm_lease_v1.h>
+#include <wlr/types/wlr_drm.h>
 #include <wlr/types/wlr_export_dmabuf_v1.h>
 #include <wlr/types/wlr_gamma_control_v1.h>
 #include <wlr/types/wlr_idle.h>
 #include <wlr/types/wlr_layer_shell_v1.h>
+#include <wlr/types/wlr_linux_dmabuf_v1.h>
 #include <wlr/types/wlr_pointer_constraints_v1.h>
 #include <wlr/types/wlr_primary_selection_v1.h>
 #include <wlr/types/wlr_relative_pointer_v1.h>
@@ -78,7 +80,13 @@ bool server_init(struct sway_server *server) {
 		return false;
 	}
 
-	wlr_renderer_init_wl_display(server->renderer, server->wl_display);
+	wlr_renderer_init_wl_shm(server->renderer, server->wl_display);
+
+	if (wlr_renderer_get_dmabuf_texture_formats(server->renderer) != NULL) {
+		wlr_drm_create(server->wl_display, server->renderer);
+		server->linux_dmabuf_v1 =
+			wlr_linux_dmabuf_v1_create(server->wl_display, server->renderer);
+	}
 
 	server->allocator = wlr_allocator_autocreate(server->backend,
 		server->renderer);

sway/tree/container.c

@@ -5,8 +5,12 @@
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
+#include <sys/stat.h>
 #include <wayland-server-core.h>
+#include <wlr/types/wlr_linux_dmabuf_v1.h>
 #include <wlr/types/wlr_output_layout.h>
+#include <wlr/render/drm_format_set.h>
+#include "linux-dmabuf-unstable-v1-protocol.h"
 #include "cairo_util.h"
 #include "pango.h"
 #include "sway/config.h"
@@ -1051,6 +1055,16 @@ void container_end_mouse_operation(struct sway_container *container) {
 	}
 }
 
+static bool devid_from_fd(int fd, dev_t *devid) {
+	struct stat stat;
+	if (fstat(fd, &stat) != 0) {
+		sway_log_errno(SWAY_ERROR, "fstat failed");
+		return false;
+	}
+	*devid = stat.st_rdev;
+	return true;
+}
+
 static void set_fullscreen(struct sway_container *con, bool enable) {
 	if (!con->view) {
 		return;
@@ -1062,6 +1076,75 @@ static void set_fullscreen(struct sway_container *con, bool enable) {
 				con->view->foreign_toplevel, enable);
 		}
 	}
+
+	if (!server.linux_dmabuf_v1 || !con->view->surface) {
+		return;
+	}
+	if (!enable) {
+		wlr_linux_dmabuf_v1_set_surface_feedback(server.linux_dmabuf_v1,
+			con->view->surface, NULL);
+		return;
+	}
+
+	if (!con->pending.workspace || !con->pending.workspace->output) {
+		return;
+	}
+
+	struct sway_output *output = con->pending.workspace->output;
+	struct wlr_output *wlr_output = output->wlr_output;
+
+	// TODO: add wlroots helpers for all of this stuff
+
+	const struct wlr_drm_format_set *renderer_formats =
+		wlr_renderer_get_dmabuf_texture_formats(server.renderer);
+	assert(renderer_formats);
+
+	int renderer_drm_fd = wlr_renderer_get_drm_fd(server.renderer);
+	int backend_drm_fd = wlr_backend_get_drm_fd(wlr_output->backend);
+	if (renderer_drm_fd < 0 || backend_drm_fd < 0) {
+		return;
+	}
+
+	dev_t render_dev, scanout_dev;
+	if (!devid_from_fd(renderer_drm_fd, &render_dev) ||
+			!devid_from_fd(backend_drm_fd, &scanout_dev)) {
+		return;
+	}
+
+	const struct wlr_drm_format_set *output_formats =
+		wlr_output_get_primary_formats(output->wlr_output,
+		WLR_BUFFER_CAP_DMABUF);
+	if (!output_formats) {
+		return;
+	}
+
+	struct wlr_drm_format_set scanout_formats = {0};
+	if (!wlr_drm_format_set_intersect(&scanout_formats,
+			output_formats, renderer_formats)) {
+		return;
+	}
+
+	struct wlr_linux_dmabuf_feedback_v1_tranche tranches[] = {
+		{
+			.target_device = scanout_dev,
+			.flags = ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT,
+			.formats = &scanout_formats,
+		},
+		{
+			.target_device = render_dev,
+			.formats = renderer_formats,
+		},
+	};
+
+	const struct wlr_linux_dmabuf_feedback_v1 feedback = {
+		.main_device = render_dev,
+		.tranches = tranches,
+		.tranches_len = sizeof(tranches) / sizeof(tranches[0]),
+	};
+	wlr_linux_dmabuf_v1_set_surface_feedback(server.linux_dmabuf_v1,
+		con->view->surface, &feedback);
+
+	wlr_drm_format_set_finish(&scanout_formats);
 }
 
 static void container_fullscreen_workspace(struct sway_container *con) {