render: allow wlr_renderer_impl.begin to fail

Make it return a bool to indicate success/failure. Adapt the
various implementations to check errors.

include/wlr/render/interface.h

@@ -22,7 +22,7 @@ struct wlr_fbox;
 struct wlr_renderer_impl {
 	bool (*bind_buffer)(struct wlr_renderer *renderer,
 		struct wlr_buffer *buffer);
-	void (*begin)(struct wlr_renderer *renderer, uint32_t width,
+	bool (*begin)(struct wlr_renderer *renderer, uint32_t width,
 		uint32_t height);
 	void (*end)(struct wlr_renderer *renderer);
 	void (*clear)(struct wlr_renderer *renderer, const float color[static 4]);

render/gles2/renderer.c

@@ -198,7 +198,7 @@ static bool gles2_bind_buffer(struct wlr_renderer *wlr_renderer,
 	return true;
 }
 
-static void gles2_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
+static bool gles2_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 		uint32_t height) {
 	struct wlr_gles2_renderer *renderer =
 		gles2_get_renderer_in_context(wlr_renderer);
@@ -219,6 +219,8 @@ static void gles2_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 	// for users to sling matricies themselves
 
 	pop_gles2_debug(renderer);
+
+	return true;
 }
 
 static void gles2_end(struct wlr_renderer *wlr_renderer) {

render/pixman/renderer.c

@@ -127,7 +127,7 @@ error_buffer:
 	return NULL;
 }
 
-static void pixman_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
+static bool pixman_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 		uint32_t height) {
 	struct wlr_pixman_renderer *renderer = get_renderer(wlr_renderer);
 	renderer->width = width;
@@ -139,9 +139,11 @@ static void pixman_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 	void *data = NULL;
 	uint32_t drm_format;
 	size_t stride;
-	wlr_buffer_begin_data_ptr_access(buffer->buffer,
+	if (!wlr_buffer_begin_data_ptr_access(buffer->buffer,
 			WLR_BUFFER_DATA_PTR_ACCESS_READ | WLR_BUFFER_DATA_PTR_ACCESS_WRITE,
-		&data, &drm_format, &stride);
+			&data, &drm_format, &stride)) {
+		return false;
+	}
 
 	// If the data pointer has changed, re-create the Pixman image. This can
 	// happen if it's a client buffer and the wl_shm_pool has been resized.
@@ -149,10 +151,18 @@ static void pixman_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 		pixman_format_code_t format = get_pixman_format_from_drm(drm_format);
 		assert(format != 0);
 
-		pixman_image_unref(buffer->image);
-		buffer->image = pixman_image_create_bits_no_clear(format,
+		pixman_image_t *image = pixman_image_create_bits_no_clear(format,
 			buffer->buffer->width, buffer->buffer->height, data, stride);
+		if (image == NULL) {
+			wlr_buffer_end_data_ptr_access(buffer->buffer);
+			return false;
 		}
+
+		pixman_image_unref(buffer->image);
+		buffer->image = image;
+	}
+
+	return true;
 }
 
 static void pixman_end(struct wlr_renderer *wlr_renderer) {

render/vulkan/renderer.c

@@ -691,14 +691,14 @@ static bool vulkan_bind_buffer(struct wlr_renderer *wlr_renderer,
 	return true;
 }
 
-static void vulkan_begin(struct wlr_renderer *wlr_renderer,
+static bool vulkan_begin(struct wlr_renderer *wlr_renderer,
 		uint32_t width, uint32_t height) {
 	struct wlr_vk_renderer *renderer = vulkan_get_renderer(wlr_renderer);
 	assert(renderer->current_render_buffer);
 
 	struct wlr_vk_command_buffer *cb = acquire_command_buffer(renderer);
 	if (cb == NULL) {
-		return;
+		return false;
 	}
 
 	assert(renderer->current_command_buffer == NULL);
@@ -707,7 +707,11 @@ static void vulkan_begin(struct wlr_renderer *wlr_renderer,
 	VkCommandBufferBeginInfo begin_info = {
 		.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
 	};
-	vkBeginCommandBuffer(cb->vk, &begin_info);
+	VkResult res = vkBeginCommandBuffer(cb->vk, &begin_info);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkBeginCommandBuffer", res);
+		return false;
+	}
 
 	// begin render pass
 	VkFramebuffer fb = renderer->current_render_buffer->framebuffer;
@@ -737,6 +741,8 @@ static void vulkan_begin(struct wlr_renderer *wlr_renderer,
 	renderer->render_width = width;
 	renderer->render_height = height;
 	renderer->bound_pipe = VK_NULL_HANDLE;
+
+	return true;
 }
 
 static void vulkan_end(struct wlr_renderer *wlr_renderer) {