renderer/opengl: invalidate intermediate FBs post render, avoid stencil if possible (#12848)

1 files changed, 38 insertions, 57 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 33e380e1..47df11c7 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -890,6 +890,16 @@ void CHyprOpenGLImpl::end() {
         popMonitorTransformEnabled();
     }
 
+    // invalidate our render FBs to signal to the driver we don't need them anymore
+    m_renderData.pCurrentMonData->mirrorFB.bind();
+    m_renderData.pCurrentMonData->mirrorFB.invalidate({GL_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0});
+    m_renderData.pCurrentMonData->mirrorSwapFB.bind();
+    m_renderData.pCurrentMonData->mirrorSwapFB.invalidate({GL_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0});
+    m_renderData.pCurrentMonData->offloadFB.bind();
+    m_renderData.pCurrentMonData->offloadFB.invalidate({GL_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0});
+    m_renderData.pCurrentMonData->offMainFB.bind();
+    m_renderData.pCurrentMonData->offMainFB.invalidate({GL_STENCIL_ATTACHMENT, GL_COLOR_ATTACHMENT0});
+
     // reset our data
     m_renderData.pMonitor.reset();
     m_renderData.mouseZoomFactor   = 1.f;
@@ -1351,8 +1361,6 @@ void CHyprOpenGLImpl::clear(const CHyprColor& color) {
             glClear(GL_COLOR_BUFFER_BIT);
         });
     }
-
-    scissor(nullptr);
 }
 
 void CHyprOpenGLImpl::blend(bool enabled) {
@@ -1432,40 +1440,15 @@ void CHyprOpenGLImpl::renderRectWithBlurInternal(const CBox& box, const CHyprCol
 
     m_renderData.currentFB->bind();
 
-    // make a stencil for rounded corners to work with blur
-    scissor(nullptr); // allow the entire window and stencil to render
-    glClearStencil(0);
-    glClear(GL_STENCIL_BUFFER_BIT);
-
-    setCapStatus(GL_STENCIL_TEST, true);
-
-    glStencilFunc(GL_ALWAYS, 1, 0xFF);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-
-    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-    renderRect(box, CHyprColor(0, 0, 0, 0), SRectRenderData{.round = data.round, .roundingPower = data.roundingPower});
-    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-
-    glStencilFunc(GL_EQUAL, 1, 0xFF);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-
-    scissor(box);
     CBox MONITORBOX = {0, 0, m_renderData.pMonitor->m_transformedSize.x, m_renderData.pMonitor->m_transformedSize.y};
     pushMonitorTransformEnabled(true);
     const auto SAVEDRENDERMODIF = m_renderData.renderModif;
     m_renderData.renderModif    = {}; // fix shit
     renderTexture(POUTFB->getTexture(), MONITORBOX,
-                  STextureRenderData{.damage = &damage, .a = data.blurA, .round = 0, .roundingPower = 2.0f, .allowCustomUV = false, .allowDim = false, .noAA = false});
+                  STextureRenderData{.damage = &damage, .a = data.blurA, .round = data.round, .roundingPower = 2.F, .allowCustomUV = false, .allowDim = false, .noAA = false});
     popMonitorTransformEnabled();
     m_renderData.renderModif = SAVEDRENDERMODIF;
 
-    glClearStencil(0);
-    glClear(GL_STENCIL_BUFFER_BIT);
-    setCapStatus(GL_STENCIL_TEST, false);
-    glStencilMask(0xFF);
-    glStencilFunc(GL_ALWAYS, 1, 0xFF);
-    scissor(nullptr);
-
     renderRectWithDamageInternal(box, col, data);
 }
 
@@ -2386,32 +2369,35 @@ void CHyprOpenGLImpl::renderTextureWithBlurInternal(SP<CTexture> tex, const CBox
 
     m_renderData.currentFB->bind();
 
-    // make a stencil for rounded corners to work with blur
-    scissor(nullptr); // allow the entire window and stencil to render
-    glClearStencil(0);
-    glClear(GL_STENCIL_BUFFER_BIT);
+    const auto NEEDS_STENCIL = m_renderData.discardMode != 0;
 
-    setCapStatus(GL_STENCIL_TEST, true);
+    if (NEEDS_STENCIL) {
+        scissor(nullptr); // allow the entire window and stencil to render
+        glClearStencil(0);
+        glClear(GL_STENCIL_BUFFER_BIT);
 
-    glStencilFunc(GL_ALWAYS, 1, 0xFF);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+        setCapStatus(GL_STENCIL_TEST, true);
 
-    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-    if (USENEWOPTIMIZE && !(m_renderData.discardMode & DISCARD_ALPHA))
-        renderRect(box, CHyprColor(0, 0, 0, 0), SRectRenderData{.round = data.round, .roundingPower = data.roundingPower});
-    else
-        renderTexture(tex, box,
-                      STextureRenderData{.a             = data.a,
-                                         .round         = data.round,
-                                         .roundingPower = data.roundingPower,
-                                         .discardActive = true,
-                                         .allowCustomUV = true,
-                                         .wrapX         = data.wrapX,
-                                         .wrapY         = data.wrapY}); // discard opaque
-    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-
-    glStencilFunc(GL_EQUAL, 1, 0xFF);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+        glStencilFunc(GL_ALWAYS, 1, 0xFF);
+        glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+
+        glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+        if (USENEWOPTIMIZE && !(m_renderData.discardMode & DISCARD_ALPHA))
+            renderRect(box, CHyprColor(0, 0, 0, 0), SRectRenderData{.round = data.round, .roundingPower = data.roundingPower});
+        else
+            renderTexture(tex, box,
+                          STextureRenderData{.a             = data.a,
+                                             .round         = data.round,
+                                             .roundingPower = data.roundingPower,
+                                             .discardActive = true,
+                                             .allowCustomUV = true,
+                                             .wrapX         = data.wrapX,
+                                             .wrapY         = data.wrapY}); // discard opaque
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+
+        glStencilFunc(GL_EQUAL, 1, 0xFF);
+        glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+    }
 
     // stencil done. Render everything.
     const auto LASTTL = m_renderData.primarySurfaceUVTopLeft;
@@ -2452,10 +2438,6 @@ void CHyprOpenGLImpl::renderTextureWithBlurInternal(SP<CTexture> tex, const CBox
     m_renderData.primarySurfaceUVTopLeft     = LASTTL;
     m_renderData.primarySurfaceUVBottomRight = LASTBR;
 
-    // render the window, but clear stencil
-    glClearStencil(0);
-    glClear(GL_STENCIL_BUFFER_BIT);
-
     // draw window
     setCapStatus(GL_STENCIL_TEST, false);
     renderTextureInternal(tex, box,
@@ -2472,8 +2454,7 @@ void CHyprOpenGLImpl::renderTextureWithBlurInternal(SP<CTexture> tex, const CBox
                               .wrapY         = data.wrapY,
                           });
 
-    glStencilMask(0xFF);
-    glStencilFunc(GL_ALWAYS, 1, 0xFF);
+    m_renderData.currentFB->invalidate({GL_STENCIL_ATTACHMENT});
     scissor(nullptr);
 }