From 0e927533b007959a13f81107babc00fe5272871b Mon Sep 17 00:00:00 2001
From: Simon Ser <contact@emersion.fr>
Date: Wed, 25 Nov 2020 16:24:07 +0100
Subject: [PATCH] backend/drm: query render formats

On some platforms it's possible that the display engine supports
modifiers not supported by the render engine.

Query render formats and intersect them with plane formats to accommodate
for this.
---
 backend/drm/renderer.c | 57 +++++++++++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 17 deletions(-)

diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index fb915fbe..7f6fe62d 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -12,6 +12,7 @@
 #include <wlr/types/wlr_matrix.h>
 #include <wlr/util/log.h>
 #include "backend/drm/drm.h"
+#include "render/drm_format_set.h"
 #include "render/gbm_allocator.h"
 #include "render/swapchain.h"
 #include "render/wlr_renderer.h"
@@ -239,39 +240,61 @@ bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 	if (!wlr_drm_format_set_has(&plane->formats, format, DRM_FORMAT_MOD_INVALID)) {
 		format = strip_alpha_channel(format);
 	}
-	if (!wlr_drm_format_set_has(&plane->formats, format, DRM_FORMAT_MOD_INVALID)) {
+	const struct wlr_drm_format *plane_format =
+		wlr_drm_format_set_get(&plane->formats, format);
+	if (plane_format == NULL) {
 		wlr_log(WLR_ERROR, "Plane %"PRIu32" doesn't support format 0x%"PRIX32,
 			plane->id, format);
 		return false;
 	}
 
-	const struct wlr_drm_format *drm_format = NULL;
-	const struct wlr_drm_format format_no_modifiers = { .format = format };
+	const struct wlr_drm_format_set *render_formats =
+		wlr_renderer_get_dmabuf_render_formats(drm->renderer.wlr_rend);
+	if (render_formats == NULL) {
+		wlr_log(WLR_ERROR, "Failed to get render formats");
+		return false;
+	}
+	const struct wlr_drm_format *render_format =
+		wlr_drm_format_set_get(render_formats, format);
+	if (render_format == NULL) {
+		wlr_log(WLR_ERROR, "Renderer doesn't support format 0x%"PRIX32,
+			format);
+		return false;
+	}
+
+	struct wlr_drm_format *drm_format = NULL;
 	if (with_modifiers) {
-		drm_format = wlr_drm_format_set_get(&plane->formats, format);
+		drm_format = wlr_drm_format_intersect(plane_format, render_format);
+		if (drm_format == NULL) {
+			wlr_log(WLR_ERROR,
+				"Failed to intersect plane and render formats 0x%"PRIX32,
+				format);
+			return false;
+		}
 	} else {
-		drm_format = &format_no_modifiers;
+		const struct wlr_drm_format format_no_modifiers = { .format = format };
+		drm_format = wlr_drm_format_dup(&format_no_modifiers);
 	}
 
 	drm_plane_finish_surface(plane);
 
+	bool ok = true;
 	if (!drm->parent) {
-		return init_drm_surface(&plane->surf, &drm->renderer, width, height,
+		ok = init_drm_surface(&plane->surf, &drm->renderer, width, height,
 			drm_format, flags | GBM_BO_USE_SCANOUT);
+	} else {
+		ok = init_drm_surface(&plane->surf, &drm->parent->renderer,
+			width, height, drm_format, flags | GBM_BO_USE_LINEAR);
+		if (ok && !init_drm_surface(&plane->mgpu_surf, &drm->renderer,
+				width, height, drm_format, flags | GBM_BO_USE_SCANOUT)) {
+			finish_drm_surface(&plane->surf);
+			ok = false;
+		}
 	}
 
-	if (!init_drm_surface(&plane->surf, &drm->parent->renderer,
-			width, height, drm_format, flags | GBM_BO_USE_LINEAR)) {
-		return false;
-	}
-
-	if (!init_drm_surface(&plane->mgpu_surf, &drm->renderer,
-			width, height, drm_format, flags | GBM_BO_USE_SCANOUT)) {
-		finish_drm_surface(&plane->surf);
-		return false;
-	}
+	free(drm_format);
 
-	return true;
+	return ok;
 }
 
 void drm_fb_clear(struct wlr_drm_fb *fb) {