backends: implement custom EGL and renderer initialization

Compositors now have more control over how the backend creates its
renderer. Currently all backends create an EGL/GLES2 renderer, so
the necessary attributes for creating the context are passed to a
user-provided callback function. It is responsible for initializing
provided wlr_egl and to return a renderer. On fail, return 0.

Fixes #987

backend/backend.c

@@ -72,8 +72,9 @@ static size_t parse_outputs_env(const char *name) {
 	return outputs;
 }
 
-static struct wlr_backend *attempt_wl_backend(struct wl_display *display) {
-	struct wlr_backend *backend = wlr_wl_backend_create(display, NULL);
+static struct wlr_backend *attempt_wl_backend(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_wl_backend_create(display, NULL, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -88,8 +89,8 @@ static struct wlr_backend *attempt_wl_backend(struct wl_display *display) {
 
 #ifdef WLR_HAS_X11_BACKEND
 static struct wlr_backend *attempt_x11_backend(struct wl_display *display,
-		const char *x11_display) {
-	struct wlr_backend *backend = wlr_x11_backend_create(display, x11_display);
+		const char *x11_display, wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_x11_backend_create(display, x11_display, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -104,8 +105,8 @@ static struct wlr_backend *attempt_x11_backend(struct wl_display *display,
 #endif
 
 static struct wlr_backend *attempt_headless_backend(
-		struct wl_display *display) {
-	struct wlr_backend *backend = wlr_headless_backend_create(display);
+		struct wl_display *display, wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_headless_backend_create(display, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -119,7 +120,8 @@ static struct wlr_backend *attempt_headless_backend(
 }
 
 static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
-		struct wlr_backend *backend, struct wlr_session *session) {
+		struct wlr_backend *backend, struct wlr_session *session,
+		wlr_renderer_create_func_t create_renderer_func) {
 	int gpus[8];
 	size_t num_gpus = wlr_session_find_gpus(session, 8, gpus);
 	struct wlr_backend *primary_drm = NULL;
@@ -127,7 +129,7 @@ static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
 
 	for (size_t i = 0; i < num_gpus; ++i) {
 		struct wlr_backend *drm = wlr_drm_backend_create(display, session,
-			gpus[i], primary_drm);
+			gpus[i], primary_drm, create_renderer_func);
 		if (!drm) {
 			wlr_log(L_ERROR, "Failed to open DRM device %d", gpus[i]);
 			continue;
@@ -145,15 +147,15 @@ static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
 
 static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 		struct wlr_backend *backend, struct wlr_session **session,
-		const char *name) {
+		const char *name, wlr_renderer_create_func_t create_renderer_func) {
 	if (strcmp(name, "wayland") == 0) {
-		return attempt_wl_backend(display);
+		return attempt_wl_backend(display, create_renderer_func);
 #ifdef WLR_HAS_X11_BACKEND
 	} else if (strcmp(name, "x11") == 0) {
-		return attempt_x11_backend(display, NULL);
+		return attempt_x11_backend(display, NULL, create_renderer_func);
 #endif
 	} else if (strcmp(name, "headless") == 0) {
-		return attempt_headless_backend(display);
+		return attempt_headless_backend(display, create_renderer_func);
 	} else if (strcmp(name, "drm") == 0 || strcmp(name, "libinput") == 0) {
 		// DRM and libinput need a session
 		*session = wlr_session_create(display);
@@ -165,7 +167,7 @@ static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 		if (strcmp(name, "libinput") == 0) {
 			return wlr_libinput_backend_create(display, *session);
 		} else {
-			return attempt_drm_backend(display, backend, *session);
+			return attempt_drm_backend(display, backend, *session, create_renderer_func);
 		}
 	}
 
@@ -173,7 +175,8 @@ static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 	return NULL;
 }
 
-struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
+struct wlr_backend *wlr_backend_autocreate(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
 	struct wlr_backend *backend = wlr_multi_backend_create(display);
 	if (!backend) {
 		wlr_log(L_ERROR, "could not allocate multibackend");
@@ -195,7 +198,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 		char *name = strtok_r(names, ",", &saveptr);
 		while (name != NULL) {
 			struct wlr_backend *subbackend =
-				attempt_backend_by_name(display, backend, &session, name);
+				attempt_backend_by_name(display, backend, &session, name, create_renderer_func);
 			if (subbackend == NULL) {
 				wlr_log(L_ERROR, "failed to start backend '%s'", name);
 				wlr_backend_destroy(backend);
@@ -218,7 +221,8 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 
 	if (getenv("WAYLAND_DISPLAY") || getenv("_WAYLAND_DISPLAY") ||
 			getenv("WAYLAND_SOCKET")) {
-		struct wlr_backend *wl_backend = attempt_wl_backend(display);
+		struct wlr_backend *wl_backend = attempt_wl_backend(display,
+			create_renderer_func);
 		if (wl_backend) {
 			wlr_multi_backend_add(backend, wl_backend);
 			return backend;
@@ -229,7 +233,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 	const char *x11_display = getenv("DISPLAY");
 	if (x11_display) {
 		struct wlr_backend *x11_backend =
-			attempt_x11_backend(display, x11_display);
+			attempt_x11_backend(display, x11_display, create_renderer_func);
 		if (x11_backend) {
 			wlr_multi_backend_add(backend, x11_backend);
 			return backend;
@@ -255,7 +259,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 	wlr_multi_backend_add(backend, libinput);
 
 	struct wlr_backend *primary_drm =
-		attempt_drm_backend(display, backend, session);
+		attempt_drm_backend(display, backend, session, create_renderer_func);
 	if (!primary_drm) {
 		wlr_log(L_ERROR, "Failed to open any DRM device");
 		wlr_backend_destroy(libinput);

backend/drm/backend.c

@@ -114,7 +114,8 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
-		struct wlr_session *session, int gpu_fd, struct wlr_backend *parent) {
+		struct wlr_session *session, int gpu_fd, struct wlr_backend *parent,
+		wlr_renderer_create_func_t create_renderer_func) {
 	assert(display && session && gpu_fd >= 0);
 	assert(!parent || wlr_backend_is_drm(parent));
 
@@ -161,7 +162,7 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 		goto error_event;
 	}
 
-	if (!init_drm_renderer(drm, &drm->renderer)) {
+	if (!init_drm_renderer(drm, &drm->renderer, create_renderer_func)) {
 		wlr_log(L_ERROR, "Failed to initialize renderer");
 		goto error_event;
 	}

backend/drm/renderer.c

@@ -19,29 +19,28 @@
 #endif
 
 bool init_drm_renderer(struct wlr_drm_backend *drm,
-		struct wlr_drm_renderer *renderer) {
+		struct wlr_drm_renderer *renderer, wlr_renderer_create_func_t create_renderer_func) {
 	renderer->gbm = gbm_create_device(drm->fd);
 	if (!renderer->gbm) {
 		wlr_log(L_ERROR, "Failed to create GBM device");
 		return false;
 	}
 
-	if (!wlr_egl_init(&renderer->egl, EGL_PLATFORM_GBM_MESA, renderer->gbm,
-			NULL, GBM_FORMAT_ARGB8888)) {
-		goto error_gbm;
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	renderer->wlr_rend = wlr_gles2_renderer_create(&renderer->egl);
+	renderer->wlr_rend = create_renderer_func(&renderer->egl,
+		EGL_PLATFORM_GBM_MESA, renderer->gbm, NULL, GBM_FORMAT_ARGB8888);
+
 	if (!renderer->wlr_rend) {
-		wlr_log(L_ERROR, "Failed to create WLR renderer");
-		goto error_egl;
+		wlr_log(L_ERROR, "Failed to create EGL/WLR renderer");
+		goto error_gbm;
 	}
 
 	renderer->fd = drm->fd;
 	return true;
 
-error_egl:
-	wlr_egl_finish(&renderer->egl);
 error_gbm:
 	gbm_device_destroy(renderer->gbm);
 	return false;

backend/headless/backend.c

@@ -78,7 +78,8 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 	backend_destroy(&backend->backend);
 }
 
-struct wlr_backend *wlr_headless_backend_create(struct wl_display *display) {
+struct wlr_backend *wlr_headless_backend_create(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
 	wlr_log(L_INFO, "Creating headless backend");
 
 	struct wlr_headless_backend *backend =
@@ -101,16 +102,18 @@ struct wlr_backend *wlr_headless_backend_create(struct wl_display *display) {
 		EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
 		EGL_NONE,
 	};
-	bool ok = wlr_egl_init(&backend->egl, EGL_PLATFORM_SURFACELESS_MESA, NULL,
-		(EGLint *)config_attribs, 0);
-	if (!ok) {
-		free(backend);
-		return NULL;
+
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	backend->renderer = wlr_gles2_renderer_create(&backend->egl);
-	if (backend->renderer == NULL) {
+	backend->renderer = create_renderer_func(&backend->egl, EGL_PLATFORM_SURFACELESS_MESA,
+		NULL, (EGLint*)config_attribs, 0);
+
+	if (!backend->renderer) {
 		wlr_log(L_ERROR, "Failed to create renderer");
+		free(backend);
+		return NULL;
 	}
 
 	backend->display_destroy.notify = handle_display_destroy;

backend/wayland/backend.c

@@ -139,7 +139,7 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
-		const char *remote) {
+		const char *remote, wlr_renderer_create_func_t create_renderer_func) {
 	wlr_log(L_INFO, "Creating wayland backend");
 
 	struct wlr_wl_backend *backend = calloc(1, sizeof(struct wlr_wl_backend));
@@ -174,14 +174,14 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
 		EGL_ALPHA_SIZE, 1,
 		EGL_NONE,
 	};
-	if (!wlr_egl_init(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
-			backend->remote_display, config_attribs, WL_SHM_FORMAT_ARGB8888)) {
-		wlr_log(L_ERROR, "Could not initialize EGL");
-		goto error_egl;
+
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
-	wlr_egl_bind_display(&backend->egl, backend->local_display);
 
-	backend->renderer = wlr_gles2_renderer_create(&backend->egl);
+	backend->renderer = create_renderer_func(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
+		backend->remote_display, config_attribs, WL_SHM_FORMAT_ARGB8888);
+
 	if (backend->renderer == NULL) {
 		wlr_log(L_ERROR, "Could not create renderer");
 		goto error_renderer;
@@ -193,8 +193,6 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
 	return &backend->backend;
 
 error_renderer:
-	wlr_egl_finish(&backend->egl);
-error_egl:
 	wl_registry_destroy(backend->registry);
 error_registry:
 	wl_display_disconnect(backend->remote_display);

backend/x11/backend.c

@@ -232,7 +232,7 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
-		const char *x11_display) {
+		const char *x11_display, wlr_renderer_create_func_t create_renderer_func) {
 	struct wlr_x11_backend *x11 = calloc(1, sizeof(*x11));
 	if (!x11) {
 		return NULL;
@@ -267,15 +267,16 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
 
 	x11->screen = xcb_setup_roots_iterator(xcb_get_setup(x11->xcb_conn)).data;
 
-	if (!wlr_egl_init(&x11->egl, EGL_PLATFORM_X11_KHR, x11->xlib_conn, NULL,
-			x11->screen->root_visual)) {
-		goto error_event;
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	x11->renderer = wlr_gles2_renderer_create(&x11->egl);
+	x11->renderer = create_renderer_func(&x11->egl, EGL_PLATFORM_X11_KHR,
+		x11->xlib_conn, NULL, x11->screen->root_visual);
+
 	if (x11->renderer == NULL) {
 		wlr_log(L_ERROR, "Failed to create renderer");
-		goto error_egl;
+		goto error_event;
 	}
 
 	wlr_input_device_init(&x11->keyboard_dev, WLR_INPUT_DEVICE_KEYBOARD,
@@ -288,8 +289,6 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
 
 	return &x11->backend;
 
-error_egl:
-	wlr_egl_finish(&x11->egl);
 error_event:
 	wl_event_source_remove(x11->event_source);
 error_x11:

examples/multi-pointer.c

@@ -284,7 +284,7 @@ int main(int argc, char *argv[]) {
 		.clear_color = { 0.25f, 0.25f, 0.25f, 1 },
 		.display = display,
 	};
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/output-layout.c

@@ -261,7 +261,7 @@ int main(int argc, char *argv[]) {
 	state.layout = wlr_output_layout_create();
 	clock_gettime(CLOCK_MONOTONIC, &state.ts_last);
 
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/pointer.c

@@ -324,7 +324,7 @@ int main(int argc, char *argv[]) {
 		.display = display
 	};
 
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/rotation.c

@@ -235,7 +235,7 @@ int main(int argc, char *argv[]) {
 	};
 	wl_list_init(&state.outputs);
 
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/simple.c

@@ -152,7 +152,7 @@ int main() {
 		.last_frame = { 0 },
 		.display = display
 	};
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/tablet.c

@@ -349,7 +349,7 @@ int main(int argc, char *argv[]) {
 	};
 	wl_list_init(&state.tablet_pads);
 	wl_list_init(&state.tablet_tools);
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

examples/touch.c

@@ -244,7 +244,7 @@ int main(int argc, char *argv[]) {
 	wl_list_init(&state.touch_points);
 	wl_list_init(&state.touch);
 
-	struct wlr_backend *wlr = wlr_backend_autocreate(display);
+	struct wlr_backend *wlr = wlr_backend_autocreate(display, NULL);
 	if (!wlr) {
 		exit(1);
 	}

include/backend/drm/renderer.h

@@ -5,6 +5,7 @@
 #include <gbm.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <wlr/backend.h>
 #include <wlr/render/wlr_renderer.h>
 
 struct wlr_drm_backend;
@@ -32,7 +33,7 @@ struct wlr_drm_surface {
 };
 
 bool init_drm_renderer(struct wlr_drm_backend *drm,
-	struct wlr_drm_renderer *renderer);
+	struct wlr_drm_renderer *renderer, wlr_renderer_create_func_t create_render);
 void finish_drm_renderer(struct wlr_drm_renderer *renderer);
 
 bool init_drm_surface(struct wlr_drm_surface *surf,

include/wlr/backend.h

@@ -20,12 +20,20 @@ struct wlr_backend {
 	} events;
 };
 
+typedef struct wlr_renderer *(*wlr_renderer_create_func_t)(struct wlr_egl *egl, EGLenum platform,
+	void *remote_display, EGLint *config_attribs, EGLint visual_id);
 /**
  * Automatically initializes the most suitable backend given the environment.
  * Will always return a multibackend. The backend is created but not started.
  * Returns NULL on failure.
+ *
+ * The compositor can request to initialize the backend's renderer by setting
+ * the create_render_func. The callback must initialize the given wlr_egl and
+ * return a valid wlr_renderer, or NULL if it has failed to initiaze it.
+ * Pass NULL as create_renderer_func to use the backend's default renderer.
  */
-struct wlr_backend *wlr_backend_autocreate(struct wl_display *display);
+struct wlr_backend *wlr_backend_autocreate(struct wl_display *display,
+	wlr_renderer_create_func_t create_renderer_func);
 /**
  * Start the backend. This may signal new_input or new_output immediately, but
  * may also wait until the display's event loop begins. Returns false on

include/wlr/backend/drm.h

@@ -14,7 +14,8 @@
  * a DRM backend, other kinds of backends raise SIGABRT).
  */
 struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
-	struct wlr_session *session, int gpu_fd, struct wlr_backend *parent);
+	struct wlr_session *session, int gpu_fd, struct wlr_backend *parent,
+	wlr_renderer_create_func_t create_renderer_func);
 
 bool wlr_backend_is_drm(struct wlr_backend *backend);
 bool wlr_output_is_drm(struct wlr_output *output);

include/wlr/backend/headless.h

@@ -9,7 +9,8 @@
  * Creates a headless backend. A headless backend has no outputs or inputs by
  * default.
  */
-struct wlr_backend *wlr_headless_backend_create(struct wl_display *display);
+struct wlr_backend *wlr_headless_backend_create(struct wl_display *display,
+	wlr_renderer_create_func_t create_renderer_func);
 /**
  * Create a new headless output backed by an in-memory EGL framebuffer. You can
  * read pixels from this framebuffer via wlr_renderer_read_pixels but it is

include/wlr/backend/wayland.h

@@ -16,7 +16,8 @@
  * to NULL for the default behaviour (WAYLAND_DISPLAY env variable or wayland-0
  * default)
  */
-struct wlr_backend *wlr_wl_backend_create(struct wl_display *display, const char *remote);
+struct wlr_backend *wlr_wl_backend_create(struct wl_display *display, const char *remote,
+	wlr_renderer_create_func_t create_renderer_func);
 
 /**
  * Adds a new output to this backend. You may remove outputs by destroying them.

include/wlr/backend/x11.h

@@ -8,7 +8,7 @@
 #include <wlr/types/wlr_output.h>
 
 struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
-	const char *x11_display);
+	const char *x11_display, wlr_renderer_create_func_t create_renderer_func);
 struct wlr_output *wlr_x11_output_create(struct wlr_backend *backend);
 
 bool wlr_backend_is_x11(struct wlr_backend *backend);

include/wlr/render/wlr_renderer.h

@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 #include <wayland-server-protocol.h>
+#include <wlr/render/egl.h>
 #include <wlr/render/wlr_texture.h>
 #include <wlr/types/wlr_box.h>
 
@@ -16,6 +17,9 @@ struct wlr_renderer {
 	} events;
 };
 
+struct wlr_renderer *wlr_renderer_autocreate(struct wlr_egl *egl, EGLenum platform,
+	void *remote_display, EGLint *config_attribs, EGLint visual_id);
+
 void wlr_renderer_begin(struct wlr_renderer *r, int width, int height);
 void wlr_renderer_end(struct wlr_renderer *r);
 void wlr_renderer_clear(struct wlr_renderer *r, const float color[static 4]);

render/wlr_renderer.c

@@ -4,6 +4,7 @@
 #include <wlr/render/interface.h>
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_matrix.h>
+#include <wlr/render/gles2.h>
 #include <wlr/util/log.h>
 #include "util/signal.h"
 
@@ -179,3 +180,19 @@ void wlr_renderer_init_wl_shm(struct wlr_renderer *r,
 		}
 	}
 }
+
+struct wlr_renderer *wlr_renderer_autocreate(struct wlr_egl *egl,
+		EGLenum platform, void *remote_display, EGLint *config_attribs, EGLint visual_id) {
+
+	if (!wlr_egl_init(egl, platform, remote_display, config_attribs, visual_id)) {
+		wlr_log(L_ERROR, "Could not initialize EGL");
+		return NULL;
+	}
+
+	struct wlr_renderer *renderer = wlr_gles2_renderer_create(egl);
+	if (!renderer) {
+		wlr_egl_finish(egl);
+	}
+
+	return renderer;
+}

rootston/main.c

@@ -21,7 +21,7 @@ int main(int argc, char **argv) {
 	server.wl_event_loop = wl_display_get_event_loop(server.wl_display);
 	assert(server.config && server.wl_display && server.wl_event_loop);
 
-	server.backend = wlr_backend_autocreate(server.wl_display);
+	server.backend = wlr_backend_autocreate(server.wl_display, NULL);
 	if (server.backend == NULL) {
 		wlr_log(L_ERROR, "could not start backend");
 		return 1;