output: add wlr_output_is_direct_scanout_allowed()

This lets compositors check whether direct scan-out is possible.
Compositors will soon be responsible for manually calling this
function.

include/wlr/types/wlr_output.h

@@ -525,6 +525,13 @@ void wlr_output_render_software_cursors(struct wlr_output *output,
  */
 const struct wlr_drm_format_set *wlr_output_get_primary_formats(
 	struct wlr_output *output, uint32_t buffer_caps);
+/**
+ * Check whether direct scan-out is allowed on the output.
+ *
+ * Direct scan-out is typically disallowed when there are software cursors or
+ * during screen capture.
+ */
+bool wlr_output_is_direct_scanout_allowed(struct wlr_output *output);
 
 
 struct wlr_output_cursor *wlr_output_cursor_create(struct wlr_output *output);

types/output/output.c

@@ -1025,3 +1025,23 @@ const struct wlr_drm_format_set *wlr_output_get_primary_formats(
 
 	return formats;
 }
+
+bool wlr_output_is_direct_scanout_allowed(struct wlr_output *output) {
+	if (output->attach_render_locks > 0) {
+		wlr_log(WLR_DEBUG, "Direct scan-out disabled by lock");
+		return false;
+	}
+
+	// If the output has at least one software cursor, reject direct scan-out
+	struct wlr_output_cursor *cursor;
+	wl_list_for_each(cursor, &output->cursors, link) {
+		if (cursor->enabled && cursor->visible &&
+				cursor != output->hardware_cursor) {
+			wlr_log(WLR_DEBUG,
+				"Direct scan-out disabled by software cursor");
+			return false;
+		}
+	}
+
+	return true;
+}