render/vulkan: add lut3d output shader option

While a corresponding pipeline is created, it is not yet used.

include/render/vulkan.h

@@ -157,6 +157,13 @@ enum wlr_vk_shader_source {
 	WLR_VK_SHADER_SOURCE_SINGLE_COLOR,
 };
 
+// Constants used to pick the color transform for the blend-to-output
+// fragment shader. Must match those in shaders/output.frag
+enum wlr_vk_output_transform {
+	WLR_VK_OUTPUT_TRANSFORM_INVERSE_SRGB = 0,
+	WLR_VK_OUTPUT_TRANSFORM_LUT3D = 1,
+};
+
 struct wlr_vk_pipeline_key {
 	struct wlr_vk_pipeline_layout_key layout;
 	enum wlr_vk_shader_source source;
@@ -182,7 +189,8 @@ struct wlr_vk_render_format_setup {
 	const struct wlr_vk_format *render_format; // used in renderpass
 	VkRenderPass render_pass;
 
-	VkPipeline output_pipe;
+	VkPipeline output_pipe_srgb;
+	VkPipeline output_pipe_lut3d;
 
 	struct wlr_vk_renderer *renderer;
 	struct wl_list pipelines; // struct wlr_vk_pipeline.link

render/vulkan/pass.c

@@ -109,7 +109,7 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 		};
 		mat3_to_mat4(final_matrix, vert_pcr_data.mat4);
 
-		bind_pipeline(pass, render_buffer->render_setup->output_pipe);
+		bind_pipeline(pass, render_buffer->render_setup->output_pipe_srgb);
 		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
 			VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
 		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,

render/vulkan/renderer.c

@@ -156,7 +156,8 @@ static void destroy_render_format_setup(struct wlr_vk_renderer *renderer,
 
 	VkDevice dev = renderer->dev->dev;
 	vkDestroyRenderPass(dev, setup->render_pass, NULL);
-	vkDestroyPipeline(dev, setup->output_pipe, NULL);
+	vkDestroyPipeline(dev, setup->output_pipe_srgb, NULL);
+	vkDestroyPipeline(dev, setup->output_pipe_lut3d, NULL);
 
 	struct wlr_vk_pipeline *pipeline, *tmp_pipeline;
 	wl_list_for_each_safe(pipeline, tmp_pipeline, &setup->pipelines, link) {
@@ -1698,10 +1699,24 @@ struct wlr_vk_pipeline *setup_get_or_create_pipeline(
 }
 
 static bool init_blend_to_output_pipeline(struct wlr_vk_renderer *renderer,
-		VkRenderPass rp, VkPipelineLayout pipe_layout, VkPipeline *pipe) {
+		VkRenderPass rp, VkPipelineLayout pipe_layout, VkPipeline *pipe,
+		enum wlr_vk_output_transform transform) {
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
+	uint32_t output_transform_type = transform;
+	VkSpecializationMapEntry spec_entry = {
+		.constantID = 0,
+		.offset = 0,
+		.size = sizeof(uint32_t),
+	};
+	VkSpecializationInfo specialization = {
+		.mapEntryCount = 1,
+		.pMapEntries = &spec_entry,
+		.dataSize = sizeof(uint32_t),
+		.pData = &output_transform_type,
+	};
+
 	VkPipelineShaderStageCreateInfo tex_stages[2] = {
 		{
 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
@@ -1714,6 +1729,7 @@ static bool init_blend_to_output_pipeline(struct wlr_vk_renderer *renderer,
 			.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 			.module = renderer->output_module,
 			.pName = "main",
+			.pSpecializationInfo = &specialization,
 		},
 	};
 
@@ -2179,7 +2195,12 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 		// this is only well defined if render pass has a 2nd subpass
 		if (!init_blend_to_output_pipeline(
 				renderer, setup->render_pass, renderer->output_pipe_layout,
-				 &setup->output_pipe)) {
+				&setup->output_pipe_lut3d, WLR_VK_OUTPUT_TRANSFORM_LUT3D)) {
+			goto error;
+		}
+		if (!init_blend_to_output_pipeline(
+			renderer, setup->render_pass, renderer->output_pipe_layout,
+			&setup->output_pipe_srgb, WLR_VK_OUTPUT_TRANSFORM_INVERSE_SRGB)) {
 			goto error;
 		}
 	} else {

render/vulkan/shaders/output.frag

@@ -1,6 +1,6 @@
 #version 450
 
-layout (input_attachment_index = 0, binding = 0) uniform subpassInput in_color;
+layout (input_attachment_index = 0, set = 0, binding = 0) uniform subpassInput in_color;
 
 layout(set = 1, binding = 0) uniform sampler3D lut_3d;
 
@@ -13,6 +13,12 @@ layout(push_constant) uniform UBO {
 	float lut_3d_scale;
 } data;
 
+layout (constant_id = 0) const int OUTPUT_TRANSFORM = 0;
+
+// Matches enum wlr_vk_output_transform
+#define OUTPUT_TRANSFORM_INVERSE_SRGB 0
+#define OUTPUT_TRANSFORM_LUT_3D 1
+
 float linear_channel_to_srgb(float x) {
 	return max(min(x * 12.92, 0.04045), 1.055 * pow(x, 1. / 2.4) - 0.055);
 }
@@ -33,10 +39,22 @@ vec4 linear_color_to_srgb(vec4 color) {
 
 void main() {
 	vec4 val = subpassLoad(in_color).rgba;
-
-	// temporary code to use the 3d look up table; to be dropped next commit
-	vec3 pos = data.lut_3d_offset + vec3(0.5,0.5,0.5) * data.lut_3d_scale;
-	val.rgb *= texture(lut_3d, pos).rgb;
-
-	out_color = linear_color_to_srgb(val);
+	if (OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_LUT_3D) {
+		if (val.a == 0) {
+			out_color = vec4(0);
+			return;
+		}
+		// Convert from pre-multiplied alpha to straight alpha
+		vec3 rgb = val.rgb / val.a;
+
+		// Apply 3D LUT
+		vec3 pos = data.lut_3d_offset + rgb * data.lut_3d_scale;
+		rgb = texture(lut_3d, pos).rgb;
+
+		// Back to pre-multiplied alpha
+		out_color = vec4(rgb * val.a, val.a);
+	} else { // OUTPUT_TRANSFORM_INVERSE_SRGB
+		// Produce post-premultiplied sRGB encoded values
+		out_color = linear_color_to_srgb(val);
+	}
 }