Add wlr_output_layer

This is based on previous work [1] [2].

This new API allows compositors to display buffers without needing to
perform rendering operations. This API can be implemented on Wayland
using subsurfaces and on DRM using KMS planes.

Compared to [1], this approach leverages wlr_addon_set to let backends
attach their own private state to layers, removes the pending
state (necessary for interop with wlr_output_commit_state()) and
enum wlr_output_layer_state_field.

[1]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/1985
[2]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/3447

include/wlr/interfaces/wlr_output.h

@@ -22,7 +22,8 @@
 	WLR_OUTPUT_STATE_SCALE | \
 	WLR_OUTPUT_STATE_TRANSFORM | \
 	WLR_OUTPUT_STATE_RENDER_FORMAT | \
-	WLR_OUTPUT_STATE_SUBPIXEL)
+	WLR_OUTPUT_STATE_SUBPIXEL | \
+	WLR_OUTPUT_STATE_LAYERS)
 
 /**
  * A backend implementation of struct wlr_output.

include/wlr/types/wlr_output.h

@@ -67,6 +67,7 @@ enum wlr_output_state_field {
 	WLR_OUTPUT_STATE_GAMMA_LUT = 1 << 7,
 	WLR_OUTPUT_STATE_RENDER_FORMAT = 1 << 8,
 	WLR_OUTPUT_STATE_SUBPIXEL = 1 << 9,
+	WLR_OUTPUT_STATE_LAYERS = 1 << 10,
 };
 
 enum wlr_output_state_mode_type {
@@ -104,6 +105,10 @@ struct wlr_output_state {
 	// only valid if WLR_OUTPUT_STATE_GAMMA_LUT
 	uint16_t *gamma_lut;
 	size_t gamma_lut_size;
+
+	// only valid if WLR_OUTPUT_STATE_LAYERS
+	struct wlr_output_layer_state *layers;
+	size_t layers_len;
 };
 
 struct wlr_output_impl;
@@ -191,6 +196,8 @@ struct wlr_output {
 	struct wlr_buffer *cursor_front_buffer;
 	int software_cursor_locks; // number of locks forcing software cursors
 
+	struct wl_list layers; // wlr_output_layer.link
+
 	struct wlr_allocator *allocator;
 	struct wlr_renderer *renderer;
 	struct wlr_swapchain *swapchain;

include/wlr/types/wlr_output_layer.h

@@ -0,0 +1,68 @@
+/*
+ * This an unstable interface of wlroots. No guarantees are made regarding the
+ * future consistency of this API.
+ */
+#ifndef WLR_USE_UNSTABLE
+#error "Add -DWLR_USE_UNSTABLE to enable unstable wlroots features"
+#endif
+
+#ifndef WLR_TYPES_WLR_OUTPUT_LAYER_H
+#define WLR_TYPES_WLR_OUTPUT_LAYER_H
+
+#include <wlr/types/wlr_output.h>
+#include <wlr/util/addon.h>
+
+/**
+ * An output layer.
+ *
+ * Output layers are displayed between the output primary buffer (see
+ * wlr_output_attach_buffer() and wlr_output_attach_render()) and the cursor
+ * buffer. They can offload some rendering work to the backend.
+ *
+ * To configure output layers, callers should call wlr_output_layer_create() to
+ * create layers, attach struct wlr_output_layer_state onto
+ * struct wlr_output_state to describe their new state, and commit the output.
+ *
+ * Backends may have arbitrary limitations when it comes to displaying output
+ * layers. Backends indicate whether or not a layer can be displayed via
+ * wlr_output_layer_state.accepted after wlr_output_test() or
+ * wlr_output_commit() is called. Compositors using the output layers API
+ * directly are expected to setup layers, call wlr_output_test(), paint the
+ * layers that the backend rejected with the renderer, then call
+ * wlr_output_commit().
+ */
+struct wlr_output_layer {
+	struct wl_list link; // wlr_output.layers
+	struct wlr_addon_set addons;
+
+	void *data;
+};
+
+/**
+ * State for an output layer.
+ */
+struct wlr_output_layer_state {
+	struct wlr_output_layer *layer;
+
+	// Buffer to display, or NULL to disable the layer
+	struct wlr_buffer *buffer;
+	// Position in output-buffer-local coordinates
+	int x, y;
+
+	// Populated by the backend after wlr_output_test() and wlr_output_commit(),
+	// indicates whether the backend has acknowledged and will take care of
+	// displaying the layer
+	bool accepted;
+};
+
+/**
+ * Create a new output layer.
+ */
+struct wlr_output_layer *wlr_output_layer_create(struct wlr_output *output);
+
+/**
+ * Destroy an output layer.
+ */
+void wlr_output_layer_destroy(struct wlr_output_layer *layer);
+
+#endif

types/meson.build

@@ -56,6 +56,7 @@ wlr_files += files(
 	'wlr_linux_dmabuf_v1.c',
 	'wlr_matrix.c',
 	'wlr_output_damage.c',
+	'wlr_output_layer.c',
 	'wlr_output_layout.c',
 	'wlr_output_management_v1.c',
 	'wlr_output_power_management_v1.c',

types/output/output.c

@@ -6,6 +6,7 @@
 #include <wlr/interfaces/wlr_output.h>
 #include <wlr/types/wlr_compositor.h>
 #include <wlr/types/wlr_matrix.h>
+#include <wlr/types/wlr_output_layer.h>
 #include <wlr/util/log.h>
 #include "render/allocator/allocator.h"
 #include "render/swapchain.h"
@@ -356,6 +357,7 @@ void wlr_output_init(struct wlr_output *output, struct wlr_backend *backend,
 	output->scale = 1;
 	output->commit_seq = 0;
 	wl_list_init(&output->cursors);
+	wl_list_init(&output->layers);
 	wl_list_init(&output->resources);
 	wl_signal_init(&output->events.frame);
 	wl_signal_init(&output->events.damage);
@@ -399,6 +401,11 @@ void wlr_output_destroy(struct wlr_output *output) {
 		wlr_output_cursor_destroy(cursor);
 	}
 
+	struct wlr_output_layer *layer, *tmp_layer;
+	wl_list_for_each_safe(layer, tmp_layer, &output->layers, link) {
+		wlr_output_layer_destroy(layer);
+	}
+
 	wlr_swapchain_destroy(output->cursor_swapchain);
 	wlr_buffer_unlock(output->cursor_front_buffer);
 
@@ -665,6 +672,12 @@ static bool output_basic_test(struct wlr_output *output,
 		return false;
 	}
 
+	if (state->committed & WLR_OUTPUT_STATE_LAYERS) {
+		for (size_t i = 0; i < state->layers_len; i++) {
+			state->layers[i].accepted = false;
+		}
+	}
+
 	return true;
 }
 
@@ -823,6 +836,15 @@ bool wlr_output_commit_state(struct wlr_output *output,
 		output->needs_frame = false;
 	}
 
+	if (pending.committed & WLR_OUTPUT_STATE_LAYERS) {
+		// Commit layer ordering
+		for (size_t i = 0; i < pending.layers_len; i++) {
+			struct wlr_output_layer *layer = pending.layers[i].layer;
+			wl_list_remove(&layer->link);
+			wl_list_insert(output->layers.prev, &layer->link);
+		}
+	}
+
 	if ((pending.committed & WLR_OUTPUT_STATE_BUFFER) &&
 			output->swapchain != NULL) {
 		wlr_swapchain_set_buffer_submitted(output->swapchain, pending.buffer);

types/wlr_output_layer.c

@@ -0,0 +1,24 @@
+#include <stdlib.h>
+#include <wlr/types/wlr_output_layer.h>
+
+struct wlr_output_layer *wlr_output_layer_create(struct wlr_output *output) {
+	struct wlr_output_layer *layer = calloc(1, sizeof(*layer));
+	if (layer == NULL) {
+		return NULL;
+	}
+
+	wl_list_insert(&output->layers, &layer->link);
+	wlr_addon_set_init(&layer->addons);
+
+	return layer;
+}
+
+void wlr_output_layer_destroy(struct wlr_output_layer *layer) {
+	if (layer == NULL) {
+		return;
+	}
+
+	wlr_addon_set_finish(&layer->addons);
+	wl_list_remove(&layer->link);
+	free(layer);
+}