output: fallback to modifier-less allocation on modeset test failure

Sometimes we allocate a buffer with modifiers but then fail to
perform a modeset with it. This can happen on Intel because of
bandwidth limitations. To mitigate this issue, it's possible to
re-allocate the buffer with modifiers.

Add the logic to do so in wlr_output.

types/wlr_output.c

@@ -480,12 +480,21 @@ static struct wlr_drm_format *output_pick_format(struct wlr_output *output,
 static void output_pending_resolution(struct wlr_output *output, int *width,
 		int *height);
 
-static bool output_create_swapchain(struct wlr_output *output) {
+/**
+ * Ensure the output has a suitable swapchain. The swapchain is re-created if
+ * necessary.
+ *
+ * If allow_modifiers is set to true, the swapchain's format may use modifiers.
+ * If set to false, the swapchain's format is guaranteed to not use modifiers.
+ */
+static bool output_create_swapchain(struct wlr_output *output,
+		bool allow_modifiers) {
 	int width, height;
 	output_pending_resolution(output, &width, &height);
 
 	if (output->swapchain != NULL && output->swapchain->width == width &&
-			output->swapchain->height == height) {
+			output->swapchain->height == height &&
+			(allow_modifiers || output->swapchain->format->len == 0)) {
 		return true;
 	}
 
@@ -514,6 +523,10 @@ static bool output_create_swapchain(struct wlr_output *output) {
 	wlr_log(WLR_DEBUG, "Choosing primary buffer format 0x%"PRIX32" for output '%s'",
 		format->format, output->name);
 
+	if (!allow_modifiers && (format->len != 1 || format->modifiers[0] != DRM_FORMAT_MOD_LINEAR)) {
+		format->len = 0;
+	}
+
 	struct wlr_swapchain *swapchain =
 		wlr_swapchain_create(allocator, width, height, format);
 	free(format);
@@ -532,7 +545,7 @@ static bool output_attach_back_buffer(struct wlr_output *output,
 		int *buffer_age) {
 	assert(output->back_buffer == NULL);
 
-	if (!output_create_swapchain(output)) {
+	if (!output_create_swapchain(output, true)) {
 		return false;
 	}
 
@@ -691,11 +704,38 @@ static bool output_ensure_buffer(struct wlr_output *output) {
 	wlr_log(WLR_DEBUG, "Attaching empty buffer to output for modeset");
 
 	if (!output_attach_empty_buffer(output)) {
-		output_clear_back_buffer(output);
+		goto error;
+	}
+	if (!output->impl->test || output->impl->test(output)) {
+		return true;
+	}
+
+	output_clear_back_buffer(output);
+	output->pending.committed &= ~WLR_OUTPUT_STATE_BUFFER;
+
+	if (output->swapchain->format->len == 0) {
 		return false;
 	}
 
+	// The test failed for a buffer which has modifiers, try disabling
+	// modifiers to see if that makes a difference.
+	wlr_log(WLR_DEBUG, "Output modeset test failed, retrying without modifiers");
+
+	if (!output_create_swapchain(output, false)) {
+		return false;
+	}
+	if (!output_attach_empty_buffer(output)) {
+		goto error;
+	}
+	if (!output->impl->test(output)) {
+		goto error;
+	}
 	return true;
+
+error:
+	output_clear_back_buffer(output);
+	output->pending.committed &= ~WLR_OUTPUT_STATE_BUFFER;
+	return false;
 }
 
 static bool output_basic_test(struct wlr_output *output) {