backend/drm: remove special linear case for cursor plane

We now properly mark the cursor plane's formats as linear-only, and we
now have a version of wlr_drm_format_intersect that handles the case of
linear-only formats and implicit modifiers.

We can remove the special drm_plane_init_surface flag we had for cursor
planes. This also allows us to use a non-linear layout for cursor planes
on drivers that support it.

Tested on amdgpu GFX9.

backend/drm/drm.c

@@ -716,7 +716,7 @@ static bool drm_connector_init_renderer(struct wlr_drm_connector *conn,
 	uint32_t format = DRM_FORMAT_ARGB8888;
 
 	bool modifiers = drm->addfb2_modifiers;
-	if (!drm_plane_init_surface(plane, drm, width, height, format, false, modifiers) ||
+	if (!drm_plane_init_surface(plane, drm, width, height, format, modifiers) ||
 			!drm_connector_pageflip_renderer(conn)) {
 		if (!modifiers) {
 			wlr_drm_conn_log(conn, WLR_ERROR, "Failed to initialize renderer:"
@@ -736,7 +736,7 @@ static bool drm_connector_init_renderer(struct wlr_drm_connector *conn,
 		crtc->pending.mode = mode;
 
 		if (!drm_plane_init_surface(plane, drm, width, height, format,
-				false, modifiers)) {
+				modifiers)) {
 			return false;
 		}
 		if (!drm_connector_pageflip_renderer(conn)) {
@@ -889,7 +889,7 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		h = ret ? 64 : h;
 
 		if (!drm_plane_init_surface(plane, drm, w, h,
-				DRM_FORMAT_ARGB8888, true, false)) {
+				DRM_FORMAT_ARGB8888, false)) {
 			wlr_drm_conn_log(conn, WLR_ERROR, "Cannot allocate cursor resources");
 			return false;
 		}

backend/drm/renderer.c

@@ -222,7 +222,7 @@ static struct wlr_drm_format *create_linear_format(uint32_t format) {
 
 bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 		struct wlr_drm_backend *drm, int32_t width, uint32_t height,
-		uint32_t format, bool force_linear, bool with_modifiers) {
+		uint32_t format, bool with_modifiers) {
 	if (!wlr_drm_format_set_has(&plane->formats, format, DRM_FORMAT_MOD_INVALID)) {
 		format = strip_alpha_channel(format);
 	}
@@ -248,30 +248,22 @@ bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 		return false;
 	}
 
-	struct wlr_drm_format *drm_format = NULL;
+	struct wlr_drm_format *format_implicit_modifier = NULL;
-	if (with_modifiers) {
+	if (!with_modifiers) {
-		drm_format = wlr_drm_format_intersect(plane_format, render_format);
+		format_implicit_modifier = wlr_drm_format_create(format);
-		if (drm_format == NULL) {
+		render_format = format_implicit_modifier;
-			wlr_log(WLR_ERROR,
-				"Failed to intersect plane and render formats 0x%"PRIX32,
-				format);
-			return false;
-		}
-	} else {
-		drm_format = wlr_drm_format_create(format);
 	}
 
-	struct wlr_drm_format *drm_format_linear = create_linear_format(format);
+	struct wlr_drm_format *drm_format =
-	if (drm_format_linear == NULL) {
+		wlr_drm_format_intersect(plane_format, render_format);
-		free(drm_format);
+	if (drm_format == NULL) {
+		wlr_log(WLR_ERROR,
+			"Failed to intersect plane and render formats 0x%"PRIX32,
+			format);
+		free(format_implicit_modifier);
 		return false;
 	}
 
-	if (force_linear) {
-		free(drm_format);
-		drm_format = wlr_drm_format_dup(drm_format_linear);
-	}
-
 	drm_plane_finish_surface(plane);
 
 	bool ok = true;
@@ -279,8 +271,17 @@ bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 		ok = init_drm_surface(&plane->surf, &drm->renderer,
 			width, height, drm_format);
 	} else {
+		struct wlr_drm_format *drm_format_linear = create_linear_format(format);
+		if (drm_format_linear == NULL) {
+			free(drm_format);
+			free(format_implicit_modifier);
+			return false;
+		}
+
 		ok = init_drm_surface(&plane->surf, &drm->parent->renderer,
 			width, height, drm_format_linear);
+		free(drm_format_linear);
+
 		if (ok && !init_drm_surface(&plane->mgpu_surf, &drm->renderer,
 				width, height, drm_format)) {
 			finish_drm_surface(&plane->surf);
@@ -288,8 +289,8 @@ bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 		}
 	}
 
-	free(drm_format_linear);
 	free(drm_format);
+	free(format_implicit_modifier);
 
 	return ok;
 }

include/backend/drm/renderer.h

@@ -60,7 +60,7 @@ bool drm_surface_render_black_frame(struct wlr_drm_surface *surf);
 
 bool drm_plane_init_surface(struct wlr_drm_plane *plane,
 		struct wlr_drm_backend *drm, int32_t width, uint32_t height,
-		uint32_t format, bool force_linear, bool with_modifiers);
+		uint32_t format, bool with_modifiers);
 void drm_plane_finish_surface(struct wlr_drm_plane *plane);
 
 #endif