renderer: Various CM fixes, part 8 of refactors (#13860)

Part 8 of Ujin's refactors.

5 files changed, 24 insertions, 16 deletions

diff --git a/src/render/shaders/glsl/blur1.glsl b/src/render/shaders/glsl/blur1.glsl
index 36e7d660..86a37d88 100644
--- a/src/render/shaders/glsl/blur1.glsl
+++ b/src/render/shaders/glsl/blur1.glsl
@@ -100,10 +100,14 @@ vec4 blur1(vec2 v_texcoord, sampler2D tex, float radius, vec2 halfpixel, int pas
     vec2 uv = v_texcoord * 2.0;
 
     vec4 sum = texture(tex, uv) * 4.0;
-    sum += texture(tex, uv - halfpixel.xy * radius);
-    sum += texture(tex, uv + halfpixel.xy * radius);
-    sum += texture(tex, uv + vec2(halfpixel.x, -halfpixel.y) * radius);
-    sum += texture(tex, uv - vec2(halfpixel.x, -halfpixel.y) * radius);
+    // Those pixels might go outside the rendered area and grab some gabage.
+    // That garbage maps to 0.0-1.0 range with UINT8 buffer and doesn't have any significant impact on the end result.
+    // FP16 garbage maps to -65,504 - 65,504 and defines the end result. Clamp it here to 0.0 - 1.0 to get the same quality outcome as with UINT8.
+    // Rerendering an undamaged area to get some insignificant color accuracy increase on blur edges isn't worth it.
+    sum += clamp(texture(tex, uv - halfpixel.xy * radius), 0.0, 1.0);
+    sum += clamp(texture(tex, uv + halfpixel.xy * radius), 0.0, 1.0);
+    sum += clamp(texture(tex, uv + vec2(halfpixel.x, -halfpixel.y) * radius), 0.0, 1.0);
+    sum += clamp(texture(tex, uv - vec2(halfpixel.x, -halfpixel.y) * radius), 0.0, 1.0);
 
     vec4 color = sum / 8.0;
 
diff --git a/src/render/shaders/glsl/cm_helpers.glsl b/src/render/shaders/glsl/cm_helpers.glsl
index 472c6ea5..0d8c68e9 100644
--- a/src/render/shaders/glsl/cm_helpers.glsl
+++ b/src/render/shaders/glsl/cm_helpers.glsl
@@ -189,14 +189,10 @@ vec4 fromLinear(vec4 color, int tf) {
 }
 
 vec4 fromLinearNit(vec4 color, int tf, vec2 range) {
-    if (tf == CM_TRANSFER_FUNCTION_EXT_LINEAR)
-        color.rgb = color.rgb / SDR_MAX_LUMINANCE;
-    else {
-        color.rgb /= max(color.a, 0.001);
-        color.rgb = (color.rgb - range[0]) / (range[1] - range[0]);
-        color.rgb = fromLinearRGB(color.rgb, tf);
-        color.rgb *= color.a;
-    }
+    color.rgb = (color.rgb - range[0] * color.a) / (range[1] - range[0]); // @gulafaran
+    color.rgb /= max(color.a, 0.001);
+    color.rgb = fromLinearRGB(color.rgb, tf);
+    color.rgb *= color.a;
     return color;
 }
 
@@ -242,7 +238,7 @@ vec4
 #endif
 #if USE_MIRROR
     // TODO HDR -> SDR tonemap
-    vec4 mirrorColor = fromLinearNit(pixColor, CM_TRANSFER_FUNCTION_GAMMA22,
+    vec4 mirrorColor = fromLinearNit(pixColor, CM_TRANSFER_FUNCTION_SRGB,
                                      srcTF == CM_TRANSFER_FUNCTION_GAMMA22 || srcTF == CM_TRANSFER_FUNCTION_SRGB ? srcTFRange : vec2(SDR_MIN_LUMINANCE, SDR_MAX_LUMINANCE));
 #endif
     pixColor = fromLinearNit(pixColor, dstTF, dstTFRange);
diff --git a/src/render/shaders/glsl/gain.glsl b/src/render/shaders/glsl/gain.glsl
index 2bdc0002..f5326d9c 100644
--- a/src/render/shaders/glsl/gain.glsl
+++ b/src/render/shaders/glsl/gain.glsl
@@ -1,4 +1,5 @@
-vec3 gain(vec3 x, float k) {
+vec3 gain(vec3 src, float k) {
+    vec3 x = clamp(src, 0.0, 1.0);
     vec3 t = step(0.5, x);
     vec3 y = mix(x, 1.0 - x, t);
     vec3 a = 0.5 * pow(2.0 * y, vec3(k));
diff --git a/src/render/shaders/glsl/surface.frag b/src/render/shaders/glsl/surface.frag
index 076ca62c..596cd8fb 100644
--- a/src/render/shaders/glsl/surface.frag
+++ b/src/render/shaders/glsl/surface.frag
@@ -103,6 +103,7 @@ void main() {
 #if USE_ROUNDING
     pixColor = rounding(pixColor, radius, roundingPower, topLeft, fullSize);
 #endif
+    pixColor *= alpha;
 #if USE_BLUR
 #if USE_DISCARD
     pixColor = mix(pixColor, vec4(mix(texture(blurredBG, v_texcoord * uvSize + uvOffset).rgb, pixColor.rgb, pixColor.a), 1.0),
@@ -112,7 +113,7 @@ void main() {
 #endif
 #endif
 
-    fragColor = pixColor * alpha;
+    fragColor = pixColor;
 #if USE_MIRROR
 #if USE_TINT
     mirrorColor.rgb = mirrorColor.rgb * tint;
@@ -121,6 +122,7 @@ void main() {
 #if USE_ROUNDING
     mirrorColor = rounding(mirrorColor, radius, roundingPower, topLeft, fullSize);
 #endif
+    mirrorColor = mirrorColor * alpha;
 #if USE_BLUR
 #if USE_DISCARD
     mirrorColor = mix(mirrorColor, vec4(mix(texture(blurredBG, v_texcoord * uvSize + uvOffset).rgb, mirrorColor.rgb, mirrorColor.a), 1.0),
@@ -130,6 +132,5 @@ void main() {
 #endif
 #endif
 
-    mirrorColor = mirrorColor * alpha;
 #endif
 }
diff --git a/src/render/shaders/glsl/tonemap.glsl b/src/render/shaders/glsl/tonemap.glsl
index a0ba24ef..dcce45a6 100644
--- a/src/render/shaders/glsl/tonemap.glsl
+++ b/src/render/shaders/glsl/tonemap.glsl
@@ -60,5 +60,11 @@ vec4 tonemap(vec4 color, mat3 dstXYZ, float maxLuminance, float dstMaxLuminance,
     // scale src to dst reference
     float refScale = dstRefLuminance / srcRefLuminance;
 
+    // kind of works but doesn't use newLum at all
     return vec4(fromLMS * toLinear(vec4(ICtCpPQInv * ICtCp, 1.0), CM_TRANSFER_FUNCTION_ST2084_PQ).rgb * HDR_MAX_LUMINANCE * refScale, color[3]);
+    // breaks with overriden monitor luminances. might be caused by incorrect imput values
+    // @gulafaran
+    // vec3 outRGB = fromLMS * toLinear(vec4(ICtCpPQInv * ICtCp, 1.0), CM_TRANSFER_FUNCTION_ST2084_PQ).rgb;
+    // outRGB *= (newLum / max(luminance, 0.0001)); // actually apply the tone mapping
+    // return vec4(clamp(outRGB * HDR_MAX_LUMINANCE * refScale, 0.0, dstMaxLuminance), color[3]);
 }