opengl/shadow: fix shadow offset rendering (#14156)

2 files changed, 38 insertions, 31 deletions

diff --git a/src/render/shaders/glsl/shadow.frag b/src/render/shaders/glsl/shadow.frag
index 19f87348..e91e9a14 100644
--- a/src/render/shaders/glsl/shadow.frag
+++ b/src/render/shaders/glsl/shadow.frag
@@ -14,6 +14,8 @@ uniform mat3  targetPrimariesXYZ;
 
 uniform vec2  topLeft;
 uniform vec2  bottomRight;
+uniform vec2  windowTopLeft;
+uniform vec2  windowBottomRight;
 uniform vec2  fullSize;
 uniform float radius;
 uniform float roundingPower;
@@ -39,7 +41,7 @@ void main() {
 #else
     fragColor =
 #endif
-        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight, thick
+        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight, windowTopLeft, windowBottomRight, thick
 #if USE_CM
                   ,
                   sourceTF, targetTF, convertMatrix, srcTFRange, dstTFRange
@@ -66,4 +68,4 @@ void main() {
     fragColor   = pixColors[0];
     mirrorColor = pixColors[1];
 #endif
-}
\ No newline at end of file
+}
diff --git a/src/render/shaders/glsl/shadow.glsl b/src/render/shaders/glsl/shadow.glsl
index 6bc92572..fdfa697c 100644
--- a/src/render/shaders/glsl/shadow.glsl
+++ b/src/render/shaders/glsl/shadow.glsl
@@ -24,13 +24,38 @@ float modifiedLength(vec2 a, float roundingPower) {
     return pow(pow(abs(a.x), roundingPower) + pow(abs(a.y), roundingPower), 1.0 / roundingPower);
 }
 
+bool pointInRoundedRect(vec2 pixCoord, vec2 tl, vec2 br, float radius, float roundingPower) {
+    if (pixCoord.x < tl.x || pixCoord.x > br.x || pixCoord.y < tl.y || pixCoord.y > br.y)
+        return false;
+
+    if (radius <= 0.0)
+        return true;
+
+    radius = min(radius, min((br.x - tl.x) * 0.5, (br.y - tl.y) * 0.5));
+
+    vec2 innerTL = tl + vec2(radius, radius);
+    vec2 innerBR = br - vec2(radius, radius);
+
+    if (pixCoord.x >= innerTL.x && pixCoord.x <= innerBR.x)
+        return true;
+
+    if (pixCoord.y >= innerTL.y && pixCoord.y <= innerBR.y)
+        return true;
+
+    vec2 delta = vec2(0.0, 0.0);
+    delta.x    = pixCoord.x < innerTL.x ? innerTL.x - pixCoord.x : pixCoord.x - innerBR.x;
+    delta.y    = pixCoord.y < innerTL.y ? innerTL.y - pixCoord.y : pixCoord.y - innerBR.y;
+
+    return distanceWithRounding(delta, roundingPower) <= radius;
+}
+
 #if USE_MIRROR
 vec4[2]
 #else
 vec4
 #endif
     getShadow(vec4 pixColor, vec2 v_texcoord, float borderRadius, float roundingPower, vec2 topLeft, vec2 fullSize, float range, float shadowPower, vec2 bottomRight,
-              float decoWidth
+              vec2 windowTopLeft, vec2 windowBottomRight, float windowRadius
 #if USE_CM
               ,
               int sourceTF, int targetTF, mat3 convertMatrix, vec2 srcTFRange, vec2 dstTFRange
@@ -65,41 +90,21 @@ vec4
     if (pixCoord[0] < topLeft[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top left
-            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, topLeft.y - pixCoord.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
             done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom left
-            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, pixCoord.y - bottomRight.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
             done = true;
         }
     } else if (pixCoord[0] > bottomRight[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top right
-            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, topLeft.y - pixCoord.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
             done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom right
-            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, pixCoord.y - bottomRight.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
             done = true;
         }
     }
@@ -118,12 +123,12 @@ vec4
         if (smallest < range) {
             // between border and max shadow distance
             pixColor[3] = pixColor[3] * pow((smallest / range), shadowPower);
-        } else {
-            // inside border or window
-            pixColor[3] = 0.0;
         }
     }
 
+    if (pointInRoundedRect(pixCoord, windowTopLeft, windowBottomRight, windowRadius, roundingPower))
+        pixColor[3] = 0.0;
+
     if (pixColor[3] == 0.0) {
         discard;
 #if USE_MIRROR
@@ -171,4 +176,4 @@ vec4
     return pixColor;
 #endif
 }
-#endif
\ No newline at end of file
+#endif