backend/drm: clip FB damage

The kernel complains when the damage exceeds the FB bounds:

    [73850.448326] i915 0000:00:02.0: [drm:drm_atomic_check_only] [PLANE:31:plane 1A] invalid damage clip 0 0 2147483647 2147483647

Make the DRM backend behave like the Wayland one and allow compositors
to damage (0, 0, INT32_MAX, INT32_MAX) to repaint everything without
needing to know the exact buffer size.

Closes: https://github.com/swaywm/sway/issues/7632

backend/drm/atomic.c

@@ -143,6 +143,32 @@ bool create_gamma_lut_blob(struct wlr_drm_backend *drm,
 	return true;
 }
 
+bool create_fb_damage_clips_blob(struct wlr_drm_backend *drm,
+		struct wlr_drm_fb *fb, const pixman_region32_t *damage, uint32_t *blob_id) {
+	if (!pixman_region32_not_empty(damage)) {
+		*blob_id = 0;
+		return true;
+	}
+
+	int width = fb->wlr_buf->width;
+	int height = fb->wlr_buf->height;
+
+	pixman_region32_t clipped;
+	pixman_region32_init(&clipped);
+	pixman_region32_intersect_rect(&clipped, damage, 0, 0, width, height);
+
+	int rects_len;
+	const pixman_box32_t *rects = pixman_region32_rectangles(&clipped, &rects_len);
+	int ret = drmModeCreatePropertyBlob(drm->fd, rects, sizeof(*rects) * rects_len, blob_id);
+	pixman_region32_fini(&clipped);
+	if (ret != 0) {
+		wlr_log_errno(WLR_ERROR, "Failed to create FB_DAMAGE_CLIPS property blob");
+		return false;
+	}
+
+	return true;
+}
+
 static uint64_t max_bpc_for_format(uint32_t format) {
 	switch (format) {
 	case DRM_FORMAT_XRGB2101010:
@@ -271,15 +297,9 @@ static bool atomic_crtc_commit(struct wlr_drm_connector *conn,
 
 	uint32_t fb_damage_clips = 0;
 	if ((state->base->committed & WLR_OUTPUT_STATE_DAMAGE) &&
-			pixman_region32_not_empty(&state->base->damage) &&
 			crtc->primary->props.fb_damage_clips != 0) {
-		int rects_len;
+		create_fb_damage_clips_blob(drm, state->primary_fb,
-		const pixman_box32_t *rects =
+			&state->base->damage, &fb_damage_clips);
-			pixman_region32_rectangles(&state->base->damage, &rects_len);
-		if (drmModeCreatePropertyBlob(drm->fd, rects,
-				sizeof(*rects) * rects_len, &fb_damage_clips) != 0) {
-			wlr_log_errno(WLR_ERROR, "Failed to create FB_DAMAGE_CLIPS property blob");
-		}
 	}
 
 	bool prev_vrr_enabled =

backend/drm/libliftoff.c

@@ -324,14 +324,9 @@ static bool crtc_commit(struct wlr_drm_connector *conn,
 
 	uint32_t fb_damage_clips = 0;
 	if ((state->base->committed & WLR_OUTPUT_STATE_DAMAGE) &&
-			pixman_region32_not_empty(&state->base->damage) &&
 			crtc->primary->props.fb_damage_clips != 0) {
-		int rects_len;
+		create_fb_damage_clips_blob(drm, state->primary_fb,
-		const pixman_box32_t *rects = pixman_region32_rectangles(&state->base->damage, &rects_len);
+			&state->base->damage, &fb_damage_clips);
-		if (drmModeCreatePropertyBlob(drm->fd, rects,
-				sizeof(*rects) * rects_len, &fb_damage_clips) != 0) {
-			wlr_log_errno(WLR_ERROR, "Failed to create FB_DAMAGE_CLIPS property blob");
-		}
 	}
 
 	bool prev_vrr_enabled =

include/backend/drm/iface.h

@@ -3,6 +3,7 @@
 
 #include <stdbool.h>
 #include <stdint.h>
+#include <pixman.h>
 #include <xf86drm.h>
 #include <xf86drmMode.h>
 
@@ -10,6 +11,7 @@ struct wlr_drm_backend;
 struct wlr_drm_connector;
 struct wlr_drm_crtc;
 struct wlr_drm_connector_state;
+struct wlr_drm_fb;
 
 // Used to provide atomic or legacy DRM functions
 struct wlr_drm_interface {
@@ -33,5 +35,7 @@ bool create_mode_blob(struct wlr_drm_backend *drm,
 	const struct wlr_drm_connector_state *state, uint32_t *blob_id);
 bool create_gamma_lut_blob(struct wlr_drm_backend *drm,
 	size_t size, const uint16_t *lut, uint32_t *blob_id);
+bool create_fb_damage_clips_blob(struct wlr_drm_backend *drm,
+	struct wlr_drm_fb *fb, const pixman_region32_t *damage, uint32_t *blob_id);
 
 #endif