renderer: simplify shadows (#14047)

6 files changed, 78 insertions, 132 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 358e4702..9883d8fc 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -2253,30 +2253,33 @@ void CHyprOpenGLImpl::renderRoundedShadow(const CBox& box, int round, float roun
     shader->setUniformFloat2(SHADER_TOP_LEFT, sc<float>(TOPLEFT.x), sc<float>(TOPLEFT.y));
     shader->setUniformFloat2(SHADER_BOTTOM_RIGHT, sc<float>(BOTTOMRIGHT.x), sc<float>(BOTTOMRIGHT.y));
     shader->setUniformFloat2(SHADER_FULL_SIZE, sc<float>(FULLSIZE.x), sc<float>(FULLSIZE.y));
-    shader->setUniformFloat(SHADER_RADIUS, range + round);
+    shader->setUniformFloat(SHADER_RADIUS, round);
     shader->setUniformFloat(SHADER_ROUNDING_POWER, roundingPower);
     shader->setUniformFloat(SHADER_RANGE, range);
     shader->setUniformFloat(SHADER_SHADOW_POWER, SHADOWPOWER);
 
     glBindVertexArray(shader->getUniformLocation(SHADER_SHADER_VAO));
 
+    CRegion drawRegion;
+
     if (g_pHyprRenderer->m_renderData.clipBox.width != 0 && g_pHyprRenderer->m_renderData.clipBox.height != 0) {
-        CRegion damageClip{g_pHyprRenderer->m_renderData.clipBox.x, g_pHyprRenderer->m_renderData.clipBox.y, g_pHyprRenderer->m_renderData.clipBox.width,
-                           g_pHyprRenderer->m_renderData.clipBox.height};
-        damageClip.intersect(g_pHyprRenderer->m_renderData.damage);
+        drawRegion = {g_pHyprRenderer->m_renderData.clipBox.x, g_pHyprRenderer->m_renderData.clipBox.y, g_pHyprRenderer->m_renderData.clipBox.width,
+                      g_pHyprRenderer->m_renderData.clipBox.height};
+        drawRegion.intersect(g_pHyprRenderer->m_renderData.damage);
+    } else
+        drawRegion = g_pHyprRenderer->m_renderData.damage;
 
-        if (!damageClip.empty()) {
-            damageClip.forEachRect([this](const auto& RECT) {
-                scissor(&RECT, g_pHyprRenderer->m_renderData.transformDamage);
-                glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
-            });
-        }
-    } else {
-        g_pHyprRenderer->m_renderData.damage.forEachRect([this](const auto& RECT) {
+    if (g_pHyprRenderer->m_renderData.currentWindow) {
+        auto PWINDOW = g_pHyprRenderer->m_renderData.currentWindow.lock();
+        shader->setUniformFloat(SHADER_THICK, PWINDOW->getRealBorderSize() + PWINDOW->rounding());
+        drawRegion.subtract(PWINDOW->surfaceLogicalBox().value().copy().scale(g_pHyprRenderer->m_renderData.pMonitor->m_scale).expand(-PWINDOW->rounding()));
+    }
+
+    if (!drawRegion.empty())
+        drawRegion.forEachRect([this](const auto& RECT) {
             scissor(&RECT, g_pHyprRenderer->m_renderData.transformDamage);
             glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
         });
-    }
 
     glBindVertexArray(0);
 }
diff --git a/src/render/decorations/CHyprDropShadowDecoration.cpp b/src/render/decorations/CHyprDropShadowDecoration.cpp
index 44a42d25..3085facb 100644
--- a/src/render/decorations/CHyprDropShadowDecoration.cpp
+++ b/src/render/decorations/CHyprDropShadowDecoration.cpp
@@ -58,17 +58,7 @@ void CHyprDropShadowDecoration::damageEntire() {
 
     applyOffset(shadowBox);
 
-    static auto PSHADOWIGNOREWINDOW = CConfigValue<Hyprlang::INT>("decoration:shadow:ignore_window");
-    const auto  ROUNDING            = PWINDOW->rounding();
-    const auto  ROUNDINGSIZE        = ROUNDING - M_SQRT1_2 * ROUNDING + 1;
-
-    CRegion     shadowRegion(shadowBox);
-    if (*PSHADOWIGNOREWINDOW) {
-        CBox surfaceBox = PWINDOW->getWindowMainSurfaceBox();
-        applyOffset(surfaceBox);
-        surfaceBox.expand(-ROUNDINGSIZE);
-        shadowRegion.subtract(CRegion(surfaceBox));
-    }
+    CRegion shadowRegion(shadowBox);
 
     for (auto const& m : g_pCompositor->m_monitors) {
         if (!g_pHyprRenderer->shouldRenderWindow(PWINDOW, m)) {
@@ -125,10 +115,9 @@ SShadowRenderData CHyprDropShadowDecoration::getRenderData(PHLMONITOR pMonitor,
 
     const auto  PWINDOW = m_window.lock();
 
-    static auto PSHADOWSIZE         = CConfigValue<Hyprlang::INT>("decoration:shadow:range");
-    static auto PSHADOWIGNOREWINDOW = CConfigValue<Hyprlang::INT>("decoration:shadow:ignore_window");
-    static auto PSHADOWSCALE        = CConfigValue<Hyprlang::FLOAT>("decoration:shadow:scale");
-    static auto PSHADOWOFFSET       = CConfigValue<Hyprlang::VEC2>("decoration:shadow:offset");
+    static auto PSHADOWSIZE   = CConfigValue<Hyprlang::INT>("decoration:shadow:range");
+    static auto PSHADOWSCALE  = CConfigValue<Hyprlang::FLOAT>("decoration:shadow:scale");
+    static auto PSHADOWOFFSET = CConfigValue<Hyprlang::VEC2>("decoration:shadow:offset");
 
     const auto  BORDERSIZE       = PWINDOW->getRealBorderSize();
     const auto  ROUNDINGBASE     = PWINDOW->rounding();
@@ -173,43 +162,11 @@ SShadowRenderData CHyprDropShadowDecoration::getRenderData(PHLMONITOR pMonitor,
 
     g_pHyprRenderer->m_renderData.currentWindow = m_window;
 
-    CBox    windowBox;
-    CRegion saveDamage;
     fullBox.scale(pMonitor->m_scale).round();
-    if (*PSHADOWIGNOREWINDOW) {
-        windowBox      = m_lastWindowBox;
-        CBox withDecos = m_lastWindowBoxWithDecos;
-
-        // get window box
-        windowBox.translate(-pMonitor->m_position + WORKSPACEOFFSET);
-        withDecos.translate(-pMonitor->m_position + WORKSPACEOFFSET);
-
-        windowBox.translate(PWINDOW->m_floatingOffset);
-        withDecos.translate(PWINDOW->m_floatingOffset);
-
-        auto scaledExtentss = withDecos.extentsFrom(windowBox);
-        scaledExtentss      = scaledExtentss * pMonitor->m_scale;
-        scaledExtentss      = scaledExtentss.round();
-
-        // add extents
-        windowBox.scale(pMonitor->m_scale).round().addExtents(scaledExtentss);
-
-        if (windowBox.width < 1 || windowBox.height < 1)
-            return {}; // prevent assert failed
-
-        saveDamage = g_pHyprRenderer->m_renderData.damage;
-
-        g_pHyprRenderer->m_renderData.damage = fullBox;
-        g_pHyprRenderer->m_renderData.damage.subtract(windowBox.copy().expand(-ROUNDING * pMonitor->m_scale)).intersect(saveDamage);
-        g_pHyprRenderer->m_renderData.renderModif.applyToRegion(g_pHyprRenderer->m_renderData.damage);
-    }
 
     return {
-        .ignoreWindow  = *PSHADOWIGNOREWINDOW,
         .valid         = true,
         .fullBox       = fullBox,
-        .windowBox     = windowBox,
-        .saveDamage    = saveDamage,
         .rounding      = ROUNDING,
         .roundingPower = ROUNDINGPOWER,
         .size          = *PSHADOWSIZE,
@@ -233,58 +190,7 @@ void CHyprDropShadowDecoration::render(PHLMONITOR pMonitor, float const& a) {
 
     g_pHyprRenderer->disableScissor();
 
-    if (data.ignoreWindow) {
-        const auto alphaFB     = g_pHyprRenderer->m_renderData.pMonitor->resources()->getUnusedWorkBuffer();
-        const auto alphaSwapFB = g_pHyprRenderer->m_renderData.pMonitor->resources()->getUnusedWorkBuffer();
-
-        CBox       monbox = {0, 0, pMonitor->m_transformedSize.x, pMonitor->m_transformedSize.y};
-
-        auto       guard = g_pHyprRenderer->bindTempFB(alphaFB); // store current FB inside guard
-
-        // TODO not needed for 8bpc and 16fp?
-        // build the matte
-        // 10-bit formats have dogshit alpha channels, so we have to use the matte to its fullest.
-        // first, clear region of interest with black (fully transparent)
-        g_pHyprRenderer->draw(CRectPassElement::SRectData{.box = data.fullBox, .color = CHyprColor(0, 0, 0, 1), .round = 0}, monbox);
-
-        // render white shadow with the alpha of the shadow color (otherwise we clear with alpha later and shit it to 2 bit)
-        drawShadowInternal(data.fullBox, data.rounding * pMonitor->m_scale, data.roundingPower, data.size * pMonitor->m_scale,
-                           CHyprColor(1, 1, 1, PWINDOW->m_realShadowColor->value().a), a);
-
-        // render black window box ("clip")
-        g_pHyprRenderer->draw(
-            CRectPassElement::SRectData{
-                .box           = data.windowBox,
-                .color         = CHyprColor(0, 0, 0, 1),
-                .round         = (data.rounding + 1 /* This fixes small pixel gaps. */) * pMonitor->m_scale,
-                .roundingPower = data.roundingPower,
-            },
-            monbox);
-
-        g_pHyprRenderer->bindFB(alphaSwapFB);
-
-        // alpha swap just has the shadow color. It will be the "texture" to render.
-        g_pHyprRenderer->draw(CRectPassElement::SRectData{.box = data.fullBox, .color = PWINDOW->m_realShadowColor->value().stripA(), .round = 0}, monbox);
-
-        guard.reset(); // restore FB
-
-        g_pHyprRenderer->pushMonitorTransformEnabled(true);
-        g_pHyprRenderer->m_renderData.renderModif.enabled = false;
-
-        g_pHyprRenderer->draw(
-            CTextureMatteElement::STextureMatteData{
-                .box = monbox,
-                .tex = alphaSwapFB->getTexture(),
-                .fb  = alphaFB,
-            },
-            {});
-
-        g_pHyprRenderer->m_renderData.renderModif.enabled = true;
-        g_pHyprRenderer->popMonitorTransformEnabled();
-
-        g_pHyprRenderer->m_renderData.damage = data.saveDamage;
-    } else
-        drawShadowInternal(data.fullBox, data.rounding * pMonitor->m_scale, data.roundingPower, data.size * pMonitor->m_scale, PWINDOW->m_realShadowColor->value(), a);
+    drawShadowInternal(data.fullBox, data.rounding * pMonitor->m_scale, data.roundingPower, data.size * pMonitor->m_scale, PWINDOW->m_realShadowColor->value(), a);
 
     reposition();
 }
diff --git a/src/render/decorations/CHyprDropShadowDecoration.hpp b/src/render/decorations/CHyprDropShadowDecoration.hpp
index d8a01edb..6cc7f499 100644
--- a/src/render/decorations/CHyprDropShadowDecoration.hpp
+++ b/src/render/decorations/CHyprDropShadowDecoration.hpp
@@ -3,14 +3,11 @@
 #include "IHyprWindowDecoration.hpp"
 
 struct SShadowRenderData {
-    bool    ignoreWindow = false;
-    bool    valid        = false;
-    CBox    fullBox;
-    CBox    windowBox;
-    CRegion saveDamage;
-    float   rounding      = 0;
-    float   roundingPower = 0;
-    int     size          = 0;
+    bool  valid = false;
+    CBox  fullBox;
+    float rounding      = 0;
+    float roundingPower = 0;
+    int   size          = 0;
 };
 
 class CHyprDropShadowDecoration : public IHyprWindowDecoration {
diff --git a/src/render/shaders/glsl/rounding.glsl b/src/render/shaders/glsl/rounding.glsl
index 61a0bb9c..64c6bcb9 100644
--- a/src/render/shaders/glsl/rounding.glsl
+++ b/src/render/shaders/glsl/rounding.glsl
@@ -4,6 +4,10 @@
 #define M_PI               3.1415926535897932384626433832795
 #define SMOOTHING_CONSTANT (M_PI / 5.34665792551)
 
+float distanceWithRounding(vec2 coords, float roundingPower) {
+    return pow(pow(coords.x, roundingPower) + pow(coords.y, roundingPower), 1.0 / roundingPower);
+}
+
 vec4 rounding(vec4 color, float radius, float roundingPower, vec2 topLeft, vec2 fullSize) {
     vec2 pixCoord = vec2(gl_FragCoord);
     pixCoord -= topLeft + fullSize * 0.5;
@@ -12,7 +16,7 @@ vec4 rounding(vec4 color, float radius, float roundingPower, vec2 topLeft, vec2
     pixCoord += vec2(1.0, 1.0) / fullSize; // center the pix don't make it top-left
 
     if (pixCoord.x + pixCoord.y > radius) {
-        float dist = pow(pow(pixCoord.x, roundingPower) + pow(pixCoord.y, roundingPower), 1.0 / roundingPower);
+        float dist = distanceWithRounding(pixCoord, roundingPower);
 
         if (dist > radius + SMOOTHING_CONSTANT)
             discard;
diff --git a/src/render/shaders/glsl/shadow.frag b/src/render/shaders/glsl/shadow.frag
index 5ba3d879..19f87348 100644
--- a/src/render/shaders/glsl/shadow.frag
+++ b/src/render/shaders/glsl/shadow.frag
@@ -19,6 +19,7 @@ uniform float radius;
 uniform float roundingPower;
 uniform float range;
 uniform float shadowPower;
+uniform float thick;
 
 #if USE_CM
 #include "cm_helpers.glsl"
@@ -38,7 +39,7 @@ void main() {
 #else
     fragColor =
 #endif
-        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight
+        getShadow(pixColor, v_texcoord, radius, roundingPower, topLeft, fullSize, range, shadowPower, bottomRight, thick
 #if USE_CM
                   ,
                   sourceTF, targetTF, convertMatrix, srcTFRange, dstTFRange
diff --git a/src/render/shaders/glsl/shadow.glsl b/src/render/shaders/glsl/shadow.glsl
index 265dba00..6bc92572 100644
--- a/src/render/shaders/glsl/shadow.glsl
+++ b/src/render/shaders/glsl/shadow.glsl
@@ -6,6 +6,7 @@
 #define SHADOW_GLSL
 
 #include "cm_helpers.glsl"
+#include "rounding.glsl"
 
 float pixAlphaRoundedDistance(float distanceToCorner, float radius, float range, float shadowPower) {
     if (distanceToCorner > radius) {
@@ -28,7 +29,8 @@ vec4[2]
 #else
 vec4
 #endif
-    getShadow(vec4 pixColor, vec2 v_texcoord, float radius, float roundingPower, vec2 topLeft, vec2 fullSize, float range, float shadowPower, vec2 bottomRight
+    getShadow(vec4 pixColor, vec2 v_texcoord, float borderRadius, float roundingPower, vec2 topLeft, vec2 fullSize, float range, float shadowPower, vec2 bottomRight,
+              float decoWidth
 #if USE_CM
               ,
               int sourceTF, int targetTF, mat3 convertMatrix, vec2 srcTFRange, vec2 dstTFRange
@@ -51,6 +53,7 @@ vec4
 #endif
 #endif
     ) {
+    float radius        = range + borderRadius;
     float originalAlpha = pixColor[3];
 
     bool  done = false;
@@ -58,29 +61,50 @@ vec4
     vec2  pixCoord = fullSize * v_texcoord;
 
     // ok, now we check the distance to a border.
-
+    // corners
     if (pixCoord[0] < topLeft[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top left
-            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
-            done        = true;
+            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, topLeft.y - pixCoord.y), roundingPower);
+            if (borderRadius > 0.0 && distance < decoWidth) {
+                pixColor[3] = 0.0;
+            } else {
+                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - topLeft, roundingPower), radius, range, shadowPower);
+            }
+            done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom left
-            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
-            done        = true;
+            float distance = distanceWithRounding(vec2(topLeft.x - pixCoord.x, pixCoord.y - bottomRight.y), roundingPower);
+            if (borderRadius > 0.0 && distance < decoWidth) {
+                pixColor[3] = 0.0;
+            } else {
+                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(topLeft[0], bottomRight[1]), roundingPower), radius, range, shadowPower);
+            }
+            done = true;
         }
     } else if (pixCoord[0] > bottomRight[0]) {
         if (pixCoord[1] < topLeft[1]) {
             // top right
-            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
-            done        = true;
+            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, topLeft.y - pixCoord.y), roundingPower);
+            if (borderRadius > 0.0 && distance < decoWidth) {
+                pixColor[3] = 0.0;
+            } else {
+                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - vec2(bottomRight[0], topLeft[1]), roundingPower), radius, range, shadowPower);
+            }
+            done = true;
         } else if (pixCoord[1] > bottomRight[1]) {
             // bottom right
-            pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
-            done        = true;
+            float distance = distanceWithRounding(vec2(pixCoord.x - bottomRight.x, pixCoord.y - bottomRight.y), roundingPower);
+            if (borderRadius > 0.0 && distance < decoWidth) {
+                pixColor[3] = 0.0;
+            } else {
+                pixColor[3] = pixColor[3] * pixAlphaRoundedDistance(modifiedLength(pixCoord - bottomRight, roundingPower), radius, range, shadowPower);
+            }
+            done = true;
         }
     }
 
+    // edges
     if (!done) {
         // distance to all straight bb borders
         float distanceT = pixCoord[1];
@@ -92,13 +116,24 @@ vec4
         float smallest = min(min(distanceT, distanceB), min(distanceL, distanceR));
 
         if (smallest < range) {
+            // between border and max shadow distance
             pixColor[3] = pixColor[3] * pow((smallest / range), shadowPower);
+        } else {
+            // inside border or window
+            pixColor[3] = 0.0;
         }
     }
 
     if (pixColor[3] == 0.0) {
         discard;
+#if USE_MIRROR
+        vec4[2] pixColors;
+        pixColors[0] = pixColor;
+        pixColors[1] = pixColor;
+        return pixColors;
+#else
         return pixColor;
+#endif
     }
 
     // premultiply