output: refuse to commit a buffer or modeset a disabled output

References: https://github.com/swaywm/wlroots/issues/1780#issuecomment-518938390

backend/drm/drm.c

@@ -446,7 +446,9 @@ static bool drm_connector_commit(struct wlr_output *output) {
 		}
 	}
 
-	if (output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+	// TODO: support modesetting with a buffer
+	if (output->pending.committed & WLR_OUTPUT_STATE_BUFFER &&
+			!(output->pending.committed & WLR_OUTPUT_STATE_MODE)) {
 		if (!drm_connector_commit_buffer(output)) {
 			return false;
 		}

include/wlr/types/wlr_output.h

@@ -224,7 +224,7 @@ void wlr_output_destroy_global(struct wlr_output *output);
  */
 struct wlr_output_mode *wlr_output_preferred_mode(struct wlr_output *output);
 /**
- * Sets the output mode. Enables the output if it's currently disabled.
+ * Sets the output mode. The output needs to be enabled.
  *
  * Mode is double-buffered state, see `wlr_output_commit`.
  */
@@ -232,7 +232,8 @@ void wlr_output_set_mode(struct wlr_output *output,
 	struct wlr_output_mode *mode);
 /**
  * Sets a custom mode on the output. If modes are available, they are preferred.
- * Setting `refresh` to zero lets the backend pick a preferred value.
+ * Setting `refresh` to zero lets the backend pick a preferred value. The
+ * output needs to be enabled.
  *
  * Custom mode is double-buffered state, see `wlr_output_commit`.
  */
@@ -274,15 +275,19 @@ void wlr_output_effective_resolution(struct wlr_output *output,
 /**
  * Attach the renderer's buffer to the output. Compositors must call this
  * function before rendering. After they are done rendering, they should call
- * `wlr_output_commit` to submit the new frame.
+ * `wlr_output_commit` to submit the new frame. The output needs to be
+ * enabled.
  *
  * If non-NULL, `buffer_age` is set to the drawing buffer age in number of
  * frames or -1 if unknown. This is useful for damage tracking.
+ *
+ * If the compositor decides not to render after calling this function, it
+ * must call wlr_output_rollback.
  */
 bool wlr_output_attach_render(struct wlr_output *output, int *buffer_age);
 /**
  * Attach a buffer to the output. Compositors should call `wlr_output_commit`
- * to submit the new frame.
+ * to submit the new frame. The output needs to be enabled.
  */
 bool wlr_output_attach_buffer(struct wlr_output *output,
 	struct wlr_buffer *buffer);
@@ -314,6 +319,10 @@ void wlr_output_set_damage(struct wlr_output *output,
  * On failure, the pending changes are rolled back.
  */
 bool wlr_output_commit(struct wlr_output *output);
+/**
+ * Discard the pending output state.
+ */
+void wlr_output_rollback(struct wlr_output *output);
 /**
  * Manually schedules a `frame` event. If a `frame` event is already pending,
  * it is a no-op.

types/wlr_output.c

@@ -483,7 +483,7 @@ bool wlr_output_commit(struct wlr_output *output) {
 	if (output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
 		if (output->frame_pending) {
 			wlr_log(WLR_ERROR, "Tried to commit a buffer while a frame is pending");
-			return false;
+			goto error;
 		}
 		if (output->idle_frame != NULL) {
 			wl_event_source_remove(output->idle_frame);
@@ -491,6 +491,20 @@ bool wlr_output_commit(struct wlr_output *output) {
 		}
 	}
 
+	bool enabled = output->enabled;
+	if (output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		enabled = output->pending.enabled;
+	}
+
+	if (!enabled && output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		wlr_log(WLR_ERROR, "Tried to commit a buffer on a disabled output");
+		goto error;
+	}
+	if (!enabled && output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		wlr_log(WLR_ERROR, "Tried to modeset a disabled output");
+		goto error;
+	}
+
 	struct timespec now;
 	clock_gettime(CLOCK_MONOTONIC, &now);
 
@@ -554,6 +568,14 @@ bool wlr_output_commit(struct wlr_output *output) {
 
 	output_state_clear(&output->pending);
 	return true;
+
+error:
+	output_state_clear(&output->pending);
+	return false;
+}
+
+void wlr_output_rollback(struct wlr_output *output) {
+	output_state_clear(&output->pending);
 }
 
 bool wlr_output_attach_buffer(struct wlr_output *output,