opengl/shadow: fix shadow offset rendering (#14156)

5 files changed, 74 insertions, 36 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 7051c10c..bf914e31 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -2259,6 +2259,9 @@ void CHyprOpenGLImpl::renderRoundedShadow(const CBox& box, int round, float roun
     shader->setUniformFloat(SHADER_ROUNDING_POWER, roundingPower);
     shader->setUniformFloat(SHADER_RANGE, range);
     shader->setUniformFloat(SHADER_SHADOW_POWER, SHADOWPOWER);
+    shader->setUniformFloat2(SHADER_WINDOW_TOP_LEFT, -1.F, -1.F);
+    shader->setUniformFloat2(SHADER_WINDOW_BOTTOM_RIGHT, -1.F, -1.F);
+    shader->setUniformFloat(SHADER_THICK, 0.F);
 
     glBindVertexArray(shader->getUniformLocation(SHADER_SHADER_VAO));
 
@@ -2272,9 +2275,33 @@ void CHyprOpenGLImpl::renderRoundedShadow(const CBox& box, int round, float roun
         drawRegion = g_pHyprRenderer->m_renderData.damage;
 
     if (g_pHyprRenderer->m_renderData.currentWindow) {
-        auto PWINDOW = g_pHyprRenderer->m_renderData.currentWindow.lock();
-        shader->setUniformFloat(SHADER_THICK, PWINDOW->getRealBorderSize() + PWINDOW->rounding());
-        drawRegion.subtract(PWINDOW->surfaceLogicalBox().value().copy().scale(g_pHyprRenderer->m_renderData.pMonitor->m_scale).expand(-PWINDOW->rounding()));
+        const auto PWINDOW = g_pHyprRenderer->m_renderData.currentWindow.lock();
+        if (PWINDOW) {
+            if (const auto WINDOWBOX = PWINDOW->surfaceLogicalBox(); WINDOWBOX.has_value()) {
+                CBox       scaledWindowBox = WINDOWBOX.value();
+
+                const auto PWORKSPACE = PWINDOW->m_workspace;
+                if (PWORKSPACE && !PWINDOW->m_pinned)
+                    scaledWindowBox.translate(PWORKSPACE->m_renderOffset->value());
+
+                scaledWindowBox.translate(PWINDOW->m_floatingOffset);
+                scaledWindowBox.translate(-m_renderData.pMonitor->m_position);
+                scaledWindowBox.scale(m_renderData.pMonitor->m_scale).round();
+                m_renderData.renderModif.applyToBox(scaledWindowBox);
+
+                const auto cutoutTopLeft     = scaledWindowBox.pos() - newBox.pos();
+                const auto cutoutBottomRight = cutoutTopLeft + scaledWindowBox.size();
+
+                float      cutoutRadius = std::max(0.F, sc<float>(PWINDOW->rounding() * m_renderData.pMonitor->m_scale));
+                cutoutRadius            = std::round(cutoutRadius * m_renderData.renderModif.combinedScale());
+
+                shader->setUniformFloat2(SHADER_WINDOW_TOP_LEFT, sc<float>(cutoutTopLeft.x), sc<float>(cutoutTopLeft.y));
+                shader->setUniformFloat2(SHADER_WINDOW_BOTTOM_RIGHT, sc<float>(cutoutBottomRight.x), sc<float>(cutoutBottomRight.y));
+                shader->setUniformFloat(SHADER_THICK, cutoutRadius);
+
+                drawRegion.subtract(scaledWindowBox.copy().expand(-sc<int>(std::round(cutoutRadius))));
+            }
+        }
     }
 
     if (!drawRegion.empty())
diff --git a/src/render/Shader.cpp b/src/render/Shader.cpp
index 14601560..3359de66 100644
--- a/src/render/Shader.cpp
+++ b/src/render/Shader.cpp
@@ -161,8 +161,10 @@ void CShader::getUniformLocations() {
         m_uniformLocations[SHADER_SHADER_VBO_POS]
         m_uniformLocations[SHADER_SHADER_VBO_UV]
         */
-    m_uniformLocations[SHADER_TOP_LEFT]     = getUniform("topLeft");
-    m_uniformLocations[SHADER_BOTTOM_RIGHT] = getUniform("bottomRight");
+    m_uniformLocations[SHADER_TOP_LEFT]            = getUniform("topLeft");
+    m_uniformLocations[SHADER_BOTTOM_RIGHT]        = getUniform("bottomRight");
+    m_uniformLocations[SHADER_WINDOW_TOP_LEFT]     = getUniform("windowTopLeft");
+    m_uniformLocations[SHADER_WINDOW_BOTTOM_RIGHT] = getUniform("windowBottomRight");
 
     // compat for screenshaders
     auto fullSize = getUniform("fullSize");
diff --git a/src/render/Shader.hpp b/src/render/Shader.hpp
index 9b097c44..84a4014c 100644
--- a/src/render/Shader.hpp
+++ b/src/render/Shader.hpp
@@ -33,6 +33,8 @@ enum eShaderUniform : uint8_t {
     SHADER_SHADER_VBO,
     SHADER_TOP_LEFT,
     SHADER_BOTTOM_RIGHT,
+    SHADER_WINDOW_TOP_LEFT,
+    SHADER_WINDOW_BOTTOM_RIGHT,
     SHADER_FULL_SIZE,
     SHADER_FULL_SIZE_UNTRANSFORMED,
     SHADER_RADIUS,
diff --git a/src/render/shaders/glsl/shadow.frag b/src/render/shaders/glsl/shadow.frag
index 19f87348..e91e9a14 100644
--- a/src/render/shaders/glsl/shadow.frag
+++ b/src/render/shaders/glsl/shadow.frag
@@ -14,6 +14,8 @@ uniform mat3  targetPrimariesXYZ;
 
 uniform vec2  topLeft;
 uniform vec2  bottomRight;
+uniform vec2  windowTopLeft;
+uniform vec2  windowBottomRight;
 uniform vec2  fullSize;
 uniform float radius;
 uniform float roundingPower;
@@ -39,7 +41,7 @@ void main() {
 #else
     fragColor =
 #endif
-        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight, thick
+        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight, windowTopLeft, windowBottomRight, thick
 #if USE_CM
                   ,
                   sourceTF, targetTF, convertMatrix, srcTFRange, dstTFRange
@@ -66,4 +68,4 @@ void main() {
     fragColor   = pixColors[0];
     mirrorColor = pixColors[1];
 #endif
-}
\ No newline at end of file
+}
diff --git a/src/render/shaders/glsl/shadow.glsl b/src/render/shaders/glsl/shadow.glsl
index 6bc92572..fdfa697c 100644
--- a/src/render/shaders/glsl/shadow.glsl
+++ b/src/render/shaders/glsl/shadow.glsl
@@ -24,13 +24,38 @@ float modifiedLength(vec2 a, float roundingPower) {
     return pow(pow(abs(a.x), roundingPower) + pow(abs(a.y), roundingPower), 1.0 / roundingPower);
 }
 
+bool pointInRoundedRect(vec2 pixCoord, vec2 tl, vec2 br, float radius, float roundingPower) {
+    if (pixCoord.x < tl.x || pixCoord.x > br.x || pixCoord.y < tl.y || pixCoord.y > br.y)
+        return false;
+
+    if (radius <= 0.0)
+        return true;
+
+    radius = min(radius, min((br.x - tl.x) * 0.5, (br.y - tl.y) * 0.5));
+
+    vec2 innerTL = tl + vec2(radius, radius);
+    vec2 innerBR = br - vec2(radius, radius);
+
+    if (pixCoord.x >= innerTL.x && pixCoord.x <= innerBR.x)
+        return true;
+
+    if (pixCoord.y >= innerTL.y && pixCoord.y <= innerBR.y)
+        return true;
+
+    vec2 delta = vec2(0.0, 0.0);
+    delta.x    = pixCoord.x < innerTL.x ? innerTL.x - pixCoord.x : pixCoord.x - innerBR.x;
+    delta.y    = pixCoord.y < innerTL.y ? innerTL.y - pixCoord.y : pixCoord.y - innerBR.y;
+
+    return distanceWithRounding(delta, roundingPower) <= radius;
+}
+
 #if USE_MIRROR
 vec4[2]
 #else
 vec4
 #endif
     getShadow(vec4 pixColor, vec2 v_texcoord, float borderRadius, float roundingPower, vec2 topLeft, vec2 fullSize, float range, float shadowPower, vec2 bottomRight,
-              float decoWidth
+              vec2 windowTopLeft, vec2 windowBottomRight, float windowRadius
 #if USE_CM
               ,
               int sourceTF, int targetTF, mat3 convertMatrix, vec2 srcTFRange, vec2 dstTFRange
@@ -65,41 +90,21 @@ vec4
     if (pixCoord[0] < topLeft[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top left
-            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, topLeft.y - pixCoord.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
             done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom left
-            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, pixCoord.y - bottomRight.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
             done = true;
         }
     } else if (pixCoord[0] > bottomRight[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top right
-            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, topLeft.y - pixCoord.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
             done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom right
-            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, pixCoord.y - bottomRight.y), roundingPower);
-            if (borderRadius > 0.0 && distance < decoWidth) {
-                pixColor[3] = 0.0;
-            } else {
-                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
-            }
+            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
             done = true;
         }
     }
@@ -118,12 +123,12 @@ vec4
         if (smallest < range) {
             // between border and max shadow distance
             pixColor[3] = pixColor[3] * pow((smallest / range), shadowPower);
-        } else {
-            // inside border or window
-            pixColor[3] = 0.0;
         }
     }
 
+    if (pointInRoundedRect(pixCoord, windowTopLeft, windowBottomRight, windowRadius, roundingPower))
+        pixColor[3] = 0.0;
+
     if (pixColor[3] == 0.0) {
         discard;
 #if USE_MIRROR
@@ -171,4 +176,4 @@ vec4
     return pixColor;
 #endif
 }
-#endif
\ No newline at end of file
+#endif