From 5adf325333602d5b1e7ccdeb122633bbc8040ace Mon Sep 17 00:00:00 2001
From: Simon Ser <contact@emersion.fr>
Date: Tue, 17 Oct 2023 12:21:08 +0200
Subject: [PATCH] render/vulkan: undo alpha premult before sRGB
 encoding/decoding

sRGB encoding/decoding needs to happen with straight alpha, not
pre-multiplied alpha.
---
 render/vulkan/shaders/output.frag  | 24 ++++++++++++++++--------
 render/vulkan/shaders/texture.frag | 24 ++++++++++++++++--------
 2 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/render/vulkan/shaders/output.frag b/render/vulkan/shaders/output.frag
index 28a8cdc5..263f3e19 100644
--- a/render/vulkan/shaders/output.frag
+++ b/render/vulkan/shaders/output.frag
@@ -5,17 +5,25 @@ layout (input_attachment_index = 0, binding = 0) uniform subpassInput in_color;
 layout(location = 0) in vec2 uv;
 layout(location = 0) out vec4 out_color;
 
-float linear_to_srgb(float x) {
+float linear_channel_to_srgb(float x) {
 	return max(min(x * 12.92, 0.04045), 1.055 * pow(x, 1. / 2.4) - 0.055);
 }
 
-void main() {
-	vec4 val = subpassLoad(in_color).rgba;
-	out_color = vec4(
-		linear_to_srgb(val.r),
-		linear_to_srgb(val.g),
-		linear_to_srgb(val.b),
-		val.a
+vec4 linear_color_to_srgb(vec4 color) {
+	if (color.a == 0) {
+		return vec4(0);
+	}
+	color.rgb /= color.a;
+	color.rgb = vec3(
+		linear_channel_to_srgb(color.r),
+		linear_channel_to_srgb(color.g),
+		linear_channel_to_srgb(color.b)
 	);
+	color.rgb *= color.a;
+	return color;
 }
 
+void main() {
+	vec4 val = subpassLoad(in_color).rgba;
+	out_color = linear_color_to_srgb(val);
+}
diff --git a/render/vulkan/shaders/texture.frag b/render/vulkan/shaders/texture.frag
index 8b5a0c8d..eab2c2d1 100644
--- a/render/vulkan/shaders/texture.frag
+++ b/render/vulkan/shaders/texture.frag
@@ -15,23 +15,31 @@ layout (constant_id = 0) const int TEXTURE_TRANSFORM = 0;
 #define TEXTURE_TRANSFORM_IDENTITY 0
 #define TEXTURE_TRANSFORM_SRGB 1
 
-float srgb_to_linear(float x) {
+float srgb_channel_to_linear(float x) {
 	return max(x / 12.92, pow((x + 0.055) / 1.055, 2.4));
 }
 
+vec4 srgb_color_to_linear(vec4 color) {
+	if (color.a == 0) {
+		return vec4(0);
+	}
+	color.rgb /= color.a;
+	color.rgb = vec3(
+		srgb_channel_to_linear(color.r),
+		srgb_channel_to_linear(color.g),
+		srgb_channel_to_linear(color.b)
+	);
+	color.rgb *= color.a;
+	return color;
+}
+
 void main() {
 	vec4 val = textureLod(tex, uv, 0);
 	if (TEXTURE_TRANSFORM == TEXTURE_TRANSFORM_SRGB) {
-		out_color = vec4(
-			srgb_to_linear(val.r),
-			srgb_to_linear(val.g),
-			srgb_to_linear(val.b),
-			val.a
-		);
+		out_color = srgb_color_to_linear(val);
 	} else { // TEXTURE_TRANSFORM_IDENTITY
 		out_color = val;
 	}
 
 	out_color *= data.alpha;
 }
-